2024-11-10 03:16:03,934 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-10 03:16:03,952 main DEBUG Took 0.015268 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-10 03:16:03,953 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-10 03:16:03,953 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-10 03:16:03,955 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-10 03:16:03,956 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 03:16:03,968 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-10 03:16:03,987 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 03:16:03,989 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 03:16:03,990 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 03:16:03,990 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 03:16:03,991 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 03:16:03,991 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 03:16:03,993 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 03:16:03,993 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 03:16:03,994 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 03:16:03,994 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 03:16:03,995 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 03:16:03,996 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 03:16:03,997 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 03:16:03,997 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 03:16:03,998 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 03:16:03,999 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 03:16:03,999 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 03:16:04,003 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 03:16:04,004 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 03:16:04,005 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 03:16:04,005 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 03:16:04,006 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 03:16:04,007 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 03:16:04,007 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 03:16:04,008 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 03:16:04,009 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-10 03:16:04,011 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 03:16:04,012 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-10 03:16:04,015 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-10 03:16:04,016 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-10 03:16:04,017 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-10 03:16:04,018 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-10 03:16:04,027 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-10 03:16:04,029 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-10 03:16:04,031 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-10 03:16:04,031 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-10 03:16:04,031 main DEBUG createAppenders(={Console}) 2024-11-10 03:16:04,032 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-10 03:16:04,032 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-10 03:16:04,033 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-10 03:16:04,033 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-10 03:16:04,034 main DEBUG OutputStream closed 2024-11-10 03:16:04,034 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-10 03:16:04,034 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-10 03:16:04,035 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-10 03:16:04,126 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-10 03:16:04,129 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-10 03:16:04,131 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-10 03:16:04,133 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-10 03:16:04,134 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-10 03:16:04,135 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-10 03:16:04,135 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-10 03:16:04,136 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-10 03:16:04,137 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-10 03:16:04,137 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-10 03:16:04,138 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-10 03:16:04,138 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-10 03:16:04,138 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-10 03:16:04,139 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-10 03:16:04,139 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-10 03:16:04,140 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-10 03:16:04,140 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-10 03:16:04,141 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-10 03:16:04,144 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-10 03:16:04,145 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-10 03:16:04,145 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-10 03:16:04,146 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-10T03:16:04,441 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0f0991c3-80e8-30ad-d408-9222f0ade083 2024-11-10 03:16:04,445 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-10 03:16:04,446 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-10T03:16:04,457 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-11-10T03:16:04,500 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-10T03:16:04,531 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0f0991c3-80e8-30ad-d408-9222f0ade083/cluster_eaab6309-33cf-8146-ba76-8df954bdc4c1, deleteOnExit=true 2024-11-10T03:16:04,532 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-10T03:16:04,533 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0f0991c3-80e8-30ad-d408-9222f0ade083/test.cache.data in system properties and HBase conf 2024-11-10T03:16:04,534 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0f0991c3-80e8-30ad-d408-9222f0ade083/hadoop.tmp.dir in system properties and HBase conf 2024-11-10T03:16:04,535 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0f0991c3-80e8-30ad-d408-9222f0ade083/hadoop.log.dir in system properties and HBase conf 2024-11-10T03:16:04,536 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0f0991c3-80e8-30ad-d408-9222f0ade083/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-10T03:16:04,537 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0f0991c3-80e8-30ad-d408-9222f0ade083/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-10T03:16:04,538 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-10T03:16:04,715 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-10T03:16:04,889 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T03:16:04,896 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0f0991c3-80e8-30ad-d408-9222f0ade083/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-10T03:16:04,897 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0f0991c3-80e8-30ad-d408-9222f0ade083/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-10T03:16:04,898 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0f0991c3-80e8-30ad-d408-9222f0ade083/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-10T03:16:04,913 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0f0991c3-80e8-30ad-d408-9222f0ade083/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-10T03:16:04,914 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0f0991c3-80e8-30ad-d408-9222f0ade083/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-10T03:16:04,914 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0f0991c3-80e8-30ad-d408-9222f0ade083/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-10T03:16:04,915 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0f0991c3-80e8-30ad-d408-9222f0ade083/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-10T03:16:04,916 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0f0991c3-80e8-30ad-d408-9222f0ade083/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-10T03:16:04,917 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0f0991c3-80e8-30ad-d408-9222f0ade083/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-10T03:16:04,917 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0f0991c3-80e8-30ad-d408-9222f0ade083/nfs.dump.dir in system properties and HBase conf 2024-11-10T03:16:04,918 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0f0991c3-80e8-30ad-d408-9222f0ade083/java.io.tmpdir in system properties and HBase conf 2024-11-10T03:16:04,919 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0f0991c3-80e8-30ad-d408-9222f0ade083/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-10T03:16:04,919 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0f0991c3-80e8-30ad-d408-9222f0ade083/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-10T03:16:04,920 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0f0991c3-80e8-30ad-d408-9222f0ade083/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-10T03:16:07,035 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-10T03:16:07,100 INFO [Time-limited test {}] log.Log(170): Logging initialized @4628ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-10T03:16:07,164 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T03:16:07,234 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-10T03:16:07,277 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-10T03:16:07,283 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-10T03:16:07,285 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-10T03:16:07,298 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T03:16:07,302 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@610d6b27{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0f0991c3-80e8-30ad-d408-9222f0ade083/hadoop.log.dir/,AVAILABLE} 2024-11-10T03:16:07,304 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@31a6a643{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-10T03:16:07,558 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7f405bdd{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0f0991c3-80e8-30ad-d408-9222f0ade083/java.io.tmpdir/jetty-localhost-41603-hadoop-hdfs-3_4_1-tests_jar-_-any-11796703931516491268/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-10T03:16:07,581 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5b829fcb{HTTP/1.1, (http/1.1)}{localhost:41603} 2024-11-10T03:16:07,581 INFO [Time-limited test {}] server.Server(415): Started @5110ms 2024-11-10T03:16:08,597 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T03:16:08,611 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-10T03:16:08,622 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-10T03:16:08,623 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-10T03:16:08,623 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-10T03:16:08,625 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@24457f8e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0f0991c3-80e8-30ad-d408-9222f0ade083/hadoop.log.dir/,AVAILABLE} 2024-11-10T03:16:08,626 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4b5418b0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-10T03:16:08,789 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@46a94ab{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0f0991c3-80e8-30ad-d408-9222f0ade083/java.io.tmpdir/jetty-localhost-34541-hadoop-hdfs-3_4_1-tests_jar-_-any-17097360191763051314/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T03:16:08,791 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@753587b3{HTTP/1.1, (http/1.1)}{localhost:34541} 2024-11-10T03:16:08,791 INFO [Time-limited test {}] server.Server(415): Started @6320ms 2024-11-10T03:16:08,876 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-10T03:16:10,160 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0f0991c3-80e8-30ad-d408-9222f0ade083/cluster_eaab6309-33cf-8146-ba76-8df954bdc4c1/data/data1/current/BP-1178049631-172.17.0.3-1731208565928/current, will proceed with Du for space computation calculation, 2024-11-10T03:16:10,160 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0f0991c3-80e8-30ad-d408-9222f0ade083/cluster_eaab6309-33cf-8146-ba76-8df954bdc4c1/data/data2/current/BP-1178049631-172.17.0.3-1731208565928/current, will proceed with Du for space computation calculation, 2024-11-10T03:16:10,195 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-10T03:16:10,245 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x764603a4c6adf133 with lease ID 0x8b2aecc7659fc289: Processing first storage report for DS-9059999b-6525-473a-934f-085b48080227 from datanode DatanodeRegistration(127.0.0.1:40501, datanodeUuid=4301a539-d672-4639-8044-4a5372b056ca, infoPort=37975, infoSecurePort=0, ipcPort=38979, storageInfo=lv=-57;cid=testClusterID;nsid=936814706;c=1731208565928) 2024-11-10T03:16:10,247 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x764603a4c6adf133 with lease ID 0x8b2aecc7659fc289: from storage DS-9059999b-6525-473a-934f-085b48080227 node DatanodeRegistration(127.0.0.1:40501, datanodeUuid=4301a539-d672-4639-8044-4a5372b056ca, infoPort=37975, infoSecurePort=0, ipcPort=38979, storageInfo=lv=-57;cid=testClusterID;nsid=936814706;c=1731208565928), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-10T03:16:10,248 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x764603a4c6adf133 with lease ID 0x8b2aecc7659fc289: Processing first storage report for DS-0441f8e6-a3be-4b6a-b737-9a1a8771aad3 from datanode DatanodeRegistration(127.0.0.1:40501, datanodeUuid=4301a539-d672-4639-8044-4a5372b056ca, infoPort=37975, infoSecurePort=0, ipcPort=38979, storageInfo=lv=-57;cid=testClusterID;nsid=936814706;c=1731208565928) 2024-11-10T03:16:10,248 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x764603a4c6adf133 with lease ID 0x8b2aecc7659fc289: from storage DS-0441f8e6-a3be-4b6a-b737-9a1a8771aad3 node DatanodeRegistration(127.0.0.1:40501, datanodeUuid=4301a539-d672-4639-8044-4a5372b056ca, infoPort=37975, infoSecurePort=0, ipcPort=38979, storageInfo=lv=-57;cid=testClusterID;nsid=936814706;c=1731208565928), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-10T03:16:10,314 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0f0991c3-80e8-30ad-d408-9222f0ade083 2024-11-10T03:16:10,385 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0f0991c3-80e8-30ad-d408-9222f0ade083/cluster_eaab6309-33cf-8146-ba76-8df954bdc4c1/zookeeper_0, clientPort=58617, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0f0991c3-80e8-30ad-d408-9222f0ade083/cluster_eaab6309-33cf-8146-ba76-8df954bdc4c1/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0f0991c3-80e8-30ad-d408-9222f0ade083/cluster_eaab6309-33cf-8146-ba76-8df954bdc4c1/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-10T03:16:10,395 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=58617 2024-11-10T03:16:10,405 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T03:16:10,408 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T03:16:10,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741825_1001 (size=7) 2024-11-10T03:16:11,057 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8 with version=8 2024-11-10T03:16:11,057 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/hbase-staging 2024-11-10T03:16:11,173 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-10T03:16:11,443 INFO [Time-limited test {}] client.ConnectionUtils(128): master/394d2222f724:0 server-side Connection retries=45 2024-11-10T03:16:11,454 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T03:16:11,455 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-10T03:16:11,461 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-10T03:16:11,461 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T03:16:11,462 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-10T03:16:11,615 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-10T03:16:11,667 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-10T03:16:11,675 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-10T03:16:11,678 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-10T03:16:11,700 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 8273 (auto-detected) 2024-11-10T03:16:11,701 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-11-10T03:16:11,718 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:37299 2024-11-10T03:16:11,740 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:37299 connecting to ZooKeeper ensemble=127.0.0.1:58617 2024-11-10T03:16:11,803 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:372990x0, quorum=127.0.0.1:58617, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-10T03:16:11,805 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:37299-0x10122be61aa0000 connected 2024-11-10T03:16:11,940 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T03:16:11,944 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T03:16:11,955 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37299-0x10122be61aa0000, quorum=127.0.0.1:58617, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T03:16:11,958 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8, hbase.cluster.distributed=false 2024-11-10T03:16:11,980 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37299-0x10122be61aa0000, quorum=127.0.0.1:58617, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-10T03:16:11,985 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37299 2024-11-10T03:16:11,985 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37299 2024-11-10T03:16:11,986 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37299 2024-11-10T03:16:11,986 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37299 2024-11-10T03:16:11,987 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37299 2024-11-10T03:16:12,089 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/394d2222f724:0 server-side Connection retries=45 2024-11-10T03:16:12,091 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T03:16:12,091 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-10T03:16:12,091 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-10T03:16:12,092 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T03:16:12,092 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-10T03:16:12,094 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-10T03:16:12,097 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-10T03:16:12,097 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:38139 2024-11-10T03:16:12,099 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38139 connecting to ZooKeeper ensemble=127.0.0.1:58617 2024-11-10T03:16:12,101 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T03:16:12,103 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T03:16:12,115 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:381390x0, quorum=127.0.0.1:58617, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-10T03:16:12,116 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:381390x0, quorum=127.0.0.1:58617, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T03:16:12,116 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38139-0x10122be61aa0001 connected 2024-11-10T03:16:12,120 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-10T03:16:12,128 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-10T03:16:12,130 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38139-0x10122be61aa0001, quorum=127.0.0.1:58617, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-10T03:16:12,136 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38139-0x10122be61aa0001, quorum=127.0.0.1:58617, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-10T03:16:12,137 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38139 2024-11-10T03:16:12,137 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38139 2024-11-10T03:16:12,138 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38139 2024-11-10T03:16:12,144 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38139 2024-11-10T03:16:12,144 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38139 2024-11-10T03:16:12,165 DEBUG [M:0;394d2222f724:37299 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;394d2222f724:37299 2024-11-10T03:16:12,166 INFO [master/394d2222f724:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/394d2222f724,37299,1731208571229 2024-11-10T03:16:12,179 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37299-0x10122be61aa0000, quorum=127.0.0.1:58617, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T03:16:12,179 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38139-0x10122be61aa0001, quorum=127.0.0.1:58617, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T03:16:12,180 DEBUG [master/394d2222f724:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37299-0x10122be61aa0000, quorum=127.0.0.1:58617, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/394d2222f724,37299,1731208571229 2024-11-10T03:16:12,210 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38139-0x10122be61aa0001, quorum=127.0.0.1:58617, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-10T03:16:12,210 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37299-0x10122be61aa0000, quorum=127.0.0.1:58617, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T03:16:12,210 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38139-0x10122be61aa0001, quorum=127.0.0.1:58617, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T03:16:12,210 DEBUG [master/394d2222f724:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37299-0x10122be61aa0000, quorum=127.0.0.1:58617, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-10T03:16:12,211 INFO [master/394d2222f724:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/394d2222f724,37299,1731208571229 from backup master directory 2024-11-10T03:16:12,220 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37299-0x10122be61aa0000, quorum=127.0.0.1:58617, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/394d2222f724,37299,1731208571229 2024-11-10T03:16:12,220 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38139-0x10122be61aa0001, quorum=127.0.0.1:58617, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T03:16:12,220 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37299-0x10122be61aa0000, quorum=127.0.0.1:58617, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T03:16:12,221 WARN [master/394d2222f724:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-10T03:16:12,221 INFO [master/394d2222f724:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=394d2222f724,37299,1731208571229 2024-11-10T03:16:12,224 INFO [master/394d2222f724:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-10T03:16:12,225 INFO [master/394d2222f724:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-10T03:16:12,289 DEBUG [master/394d2222f724:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/hbase.id] with ID: 7f5c6727-65f7-4659-afe8-8e9a55664c78 2024-11-10T03:16:12,289 DEBUG [master/394d2222f724:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/.tmp/hbase.id 2024-11-10T03:16:12,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741826_1002 (size=42) 2024-11-10T03:16:12,302 DEBUG [master/394d2222f724:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/.tmp/hbase.id]:[hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/hbase.id] 2024-11-10T03:16:12,348 INFO [master/394d2222f724:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T03:16:12,352 INFO [master/394d2222f724:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-10T03:16:12,371 INFO [master/394d2222f724:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 15ms. 2024-11-10T03:16:12,378 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38139-0x10122be61aa0001, quorum=127.0.0.1:58617, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T03:16:12,378 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37299-0x10122be61aa0000, quorum=127.0.0.1:58617, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T03:16:12,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741827_1003 (size=196) 2024-11-10T03:16:12,408 INFO [master/394d2222f724:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T03:16:12,411 INFO [master/394d2222f724:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-10T03:16:12,431 DEBUG [master/394d2222f724:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:12,437 INFO [master/394d2222f724:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-10T03:16:12,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741828_1004 (size=1189) 2024-11-10T03:16:12,889 INFO [master/394d2222f724:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/MasterData/data/master/store 2024-11-10T03:16:12,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741829_1005 (size=34) 2024-11-10T03:16:13,318 INFO [master/394d2222f724:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-10T03:16:13,322 DEBUG [master/394d2222f724:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T03:16:13,323 DEBUG [master/394d2222f724:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-10T03:16:13,323 INFO [master/394d2222f724:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T03:16:13,323 DEBUG [master/394d2222f724:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T03:16:13,325 DEBUG [master/394d2222f724:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-10T03:16:13,325 DEBUG [master/394d2222f724:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T03:16:13,325 INFO [master/394d2222f724:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T03:16:13,326 DEBUG [master/394d2222f724:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731208573323Disabling compacts and flushes for region at 1731208573323Disabling writes for close at 1731208573325 (+2 ms)Writing region close event to WAL at 1731208573325Closed at 1731208573325 2024-11-10T03:16:13,328 WARN [master/394d2222f724:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/MasterData/data/master/store/.initializing 2024-11-10T03:16:13,328 DEBUG [master/394d2222f724:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/MasterData/WALs/394d2222f724,37299,1731208571229 2024-11-10T03:16:13,336 INFO [master/394d2222f724:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-10T03:16:13,351 INFO [master/394d2222f724:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=394d2222f724%2C37299%2C1731208571229, suffix=, logDir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/MasterData/WALs/394d2222f724,37299,1731208571229, archiveDir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/MasterData/oldWALs, maxLogs=10 2024-11-10T03:16:13,372 DEBUG [master/394d2222f724:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/MasterData/WALs/394d2222f724,37299,1731208571229/394d2222f724%2C37299%2C1731208571229.1731208573355, exclude list is [], retry=0 2024-11-10T03:16:13,387 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40501,DS-9059999b-6525-473a-934f-085b48080227,DISK] 2024-11-10T03:16:13,390 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-10T03:16:13,423 INFO [master/394d2222f724:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/MasterData/WALs/394d2222f724,37299,1731208571229/394d2222f724%2C37299%2C1731208571229.1731208573355 2024-11-10T03:16:13,424 DEBUG [master/394d2222f724:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37975:37975)] 2024-11-10T03:16:13,424 DEBUG [master/394d2222f724:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-10T03:16:13,425 DEBUG [master/394d2222f724:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T03:16:13,427 DEBUG [master/394d2222f724:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:16:13,428 DEBUG [master/394d2222f724:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:16:13,461 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:16:13,485 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-10T03:16:13,488 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:13,491 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T03:16:13,491 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:16:13,495 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-10T03:16:13,495 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:13,496 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T03:16:13,496 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:16:13,499 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-10T03:16:13,499 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:13,500 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T03:16:13,501 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:16:13,504 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-10T03:16:13,504 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:13,505 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T03:16:13,505 DEBUG [master/394d2222f724:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:16:13,509 DEBUG [master/394d2222f724:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:16:13,510 DEBUG [master/394d2222f724:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:16:13,514 DEBUG [master/394d2222f724:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:16:13,515 DEBUG [master/394d2222f724:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:16:13,519 DEBUG [master/394d2222f724:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-10T03:16:13,524 DEBUG [master/394d2222f724:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:16:13,529 DEBUG [master/394d2222f724:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T03:16:13,530 INFO [master/394d2222f724:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65668607, jitterRate=-0.021461501717567444}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-10T03:16:13,539 DEBUG [master/394d2222f724:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731208573438Initializing all the Stores at 1731208573441 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731208573441Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731208573442 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731208573442Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731208573442Cleaning up temporary data from old regions at 1731208573515 (+73 ms)Region opened successfully at 1731208573539 (+24 ms) 2024-11-10T03:16:13,541 INFO [master/394d2222f724:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-10T03:16:13,583 DEBUG [master/394d2222f724:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@258ae66f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=394d2222f724/172.17.0.3:0 2024-11-10T03:16:13,610 INFO [master/394d2222f724:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-10T03:16:13,619 INFO [master/394d2222f724:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-10T03:16:13,619 INFO [master/394d2222f724:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-10T03:16:13,623 INFO [master/394d2222f724:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-10T03:16:13,625 INFO [master/394d2222f724:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-10T03:16:13,630 INFO [master/394d2222f724:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-10T03:16:13,630 INFO [master/394d2222f724:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-10T03:16:13,660 INFO [master/394d2222f724:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-10T03:16:13,671 DEBUG [master/394d2222f724:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37299-0x10122be61aa0000, quorum=127.0.0.1:58617, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-10T03:16:13,723 DEBUG [master/394d2222f724:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-10T03:16:13,729 INFO [master/394d2222f724:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-10T03:16:13,732 DEBUG [master/394d2222f724:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37299-0x10122be61aa0000, quorum=127.0.0.1:58617, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-10T03:16:13,744 DEBUG [master/394d2222f724:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-10T03:16:13,747 INFO [master/394d2222f724:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-10T03:16:13,750 DEBUG [master/394d2222f724:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37299-0x10122be61aa0000, quorum=127.0.0.1:58617, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-10T03:16:13,756 DEBUG [master/394d2222f724:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-10T03:16:13,758 DEBUG [master/394d2222f724:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37299-0x10122be61aa0000, quorum=127.0.0.1:58617, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-10T03:16:13,767 DEBUG [master/394d2222f724:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-10T03:16:13,792 DEBUG [master/394d2222f724:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37299-0x10122be61aa0000, quorum=127.0.0.1:58617, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-10T03:16:13,799 DEBUG [master/394d2222f724:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-10T03:16:13,809 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37299-0x10122be61aa0000, quorum=127.0.0.1:58617, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-10T03:16:13,809 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38139-0x10122be61aa0001, quorum=127.0.0.1:58617, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-10T03:16:13,810 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37299-0x10122be61aa0000, quorum=127.0.0.1:58617, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T03:16:13,810 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38139-0x10122be61aa0001, quorum=127.0.0.1:58617, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T03:16:13,813 INFO [master/394d2222f724:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=394d2222f724,37299,1731208571229, sessionid=0x10122be61aa0000, setting cluster-up flag (Was=false) 2024-11-10T03:16:13,841 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37299-0x10122be61aa0000, quorum=127.0.0.1:58617, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T03:16:13,841 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38139-0x10122be61aa0001, quorum=127.0.0.1:58617, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T03:16:13,872 DEBUG [master/394d2222f724:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-10T03:16:13,875 DEBUG [master/394d2222f724:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=394d2222f724,37299,1731208571229 2024-11-10T03:16:13,894 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37299-0x10122be61aa0000, quorum=127.0.0.1:58617, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T03:16:13,894 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38139-0x10122be61aa0001, quorum=127.0.0.1:58617, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T03:16:13,925 DEBUG [master/394d2222f724:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-10T03:16:13,927 DEBUG [master/394d2222f724:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=394d2222f724,37299,1731208571229 2024-11-10T03:16:13,933 INFO [master/394d2222f724:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-10T03:16:13,950 INFO [RS:0;394d2222f724:38139 {}] regionserver.HRegionServer(746): ClusterId : 7f5c6727-65f7-4659-afe8-8e9a55664c78 2024-11-10T03:16:13,952 DEBUG [RS:0;394d2222f724:38139 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-10T03:16:13,967 DEBUG [RS:0;394d2222f724:38139 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-10T03:16:13,968 DEBUG [RS:0;394d2222f724:38139 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-10T03:16:13,979 DEBUG [RS:0;394d2222f724:38139 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-10T03:16:13,980 DEBUG [RS:0;394d2222f724:38139 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@631848d8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=394d2222f724/172.17.0.3:0 2024-11-10T03:16:13,993 DEBUG [RS:0;394d2222f724:38139 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;394d2222f724:38139 2024-11-10T03:16:13,996 INFO [RS:0;394d2222f724:38139 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-10T03:16:13,996 INFO [RS:0;394d2222f724:38139 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-10T03:16:13,996 DEBUG [RS:0;394d2222f724:38139 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-10T03:16:13,998 INFO [RS:0;394d2222f724:38139 {}] regionserver.HRegionServer(2659): reportForDuty to master=394d2222f724,37299,1731208571229 with port=38139, startcode=1731208572054 2024-11-10T03:16:14,008 DEBUG [RS:0;394d2222f724:38139 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-10T03:16:14,008 DEBUG [master/394d2222f724:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-10T03:16:14,016 INFO [master/394d2222f724:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-10T03:16:14,022 INFO [master/394d2222f724:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-10T03:16:14,027 DEBUG [master/394d2222f724:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 394d2222f724,37299,1731208571229 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-10T03:16:14,034 DEBUG [master/394d2222f724:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/394d2222f724:0, corePoolSize=5, maxPoolSize=5 2024-11-10T03:16:14,034 DEBUG [master/394d2222f724:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/394d2222f724:0, corePoolSize=5, maxPoolSize=5 2024-11-10T03:16:14,035 DEBUG [master/394d2222f724:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/394d2222f724:0, corePoolSize=5, maxPoolSize=5 2024-11-10T03:16:14,035 DEBUG [master/394d2222f724:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/394d2222f724:0, corePoolSize=5, maxPoolSize=5 2024-11-10T03:16:14,035 DEBUG [master/394d2222f724:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/394d2222f724:0, corePoolSize=10, maxPoolSize=10 2024-11-10T03:16:14,035 DEBUG [master/394d2222f724:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/394d2222f724:0, corePoolSize=1, maxPoolSize=1 2024-11-10T03:16:14,035 DEBUG [master/394d2222f724:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/394d2222f724:0, corePoolSize=2, maxPoolSize=2 2024-11-10T03:16:14,036 DEBUG [master/394d2222f724:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/394d2222f724:0, corePoolSize=1, maxPoolSize=1 2024-11-10T03:16:14,039 INFO [master/394d2222f724:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731208604039 2024-11-10T03:16:14,041 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-10T03:16:14,041 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48739, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-10T03:16:14,041 INFO [master/394d2222f724:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-10T03:16:14,041 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-10T03:16:14,042 INFO [master/394d2222f724:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-10T03:16:14,046 INFO [master/394d2222f724:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-10T03:16:14,047 INFO [master/394d2222f724:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-10T03:16:14,047 INFO [master/394d2222f724:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-10T03:16:14,048 INFO [master/394d2222f724:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-10T03:16:14,048 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:14,048 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-10T03:16:14,049 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37299 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:14,048 INFO [master/394d2222f724:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-10T03:16:14,052 INFO [master/394d2222f724:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-10T03:16:14,053 INFO [master/394d2222f724:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-10T03:16:14,054 INFO [master/394d2222f724:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-10T03:16:14,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741831_1007 (size=1321) 2024-11-10T03:16:14,064 INFO [master/394d2222f724:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-10T03:16:14,064 INFO [master/394d2222f724:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-10T03:16:14,066 DEBUG [master/394d2222f724:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/394d2222f724:0:becomeActiveMaster-HFileCleaner.large.0-1731208574065,5,FailOnTimeoutGroup] 2024-11-10T03:16:14,066 DEBUG [master/394d2222f724:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/394d2222f724:0:becomeActiveMaster-HFileCleaner.small.0-1731208574066,5,FailOnTimeoutGroup] 2024-11-10T03:16:14,067 INFO [master/394d2222f724:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-10T03:16:14,067 INFO [master/394d2222f724:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-10T03:16:14,068 INFO [master/394d2222f724:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-10T03:16:14,068 INFO [master/394d2222f724:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-10T03:16:14,076 DEBUG [RS:0;394d2222f724:38139 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-10T03:16:14,076 WARN [RS:0;394d2222f724:38139 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-10T03:16:14,178 INFO [RS:0;394d2222f724:38139 {}] regionserver.HRegionServer(2659): reportForDuty to master=394d2222f724,37299,1731208571229 with port=38139, startcode=1731208572054 2024-11-10T03:16:14,181 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37299 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 394d2222f724,38139,1731208572054 2024-11-10T03:16:14,184 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37299 {}] master.ServerManager(517): Registering regionserver=394d2222f724,38139,1731208572054 2024-11-10T03:16:14,195 DEBUG [RS:0;394d2222f724:38139 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8 2024-11-10T03:16:14,195 DEBUG [RS:0;394d2222f724:38139 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:46291 2024-11-10T03:16:14,195 DEBUG [RS:0;394d2222f724:38139 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-10T03:16:14,209 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37299-0x10122be61aa0000, quorum=127.0.0.1:58617, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-10T03:16:14,210 DEBUG [RS:0;394d2222f724:38139 {}] zookeeper.ZKUtil(111): regionserver:38139-0x10122be61aa0001, quorum=127.0.0.1:58617, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/394d2222f724,38139,1731208572054 2024-11-10T03:16:14,210 WARN [RS:0;394d2222f724:38139 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-10T03:16:14,211 INFO [RS:0;394d2222f724:38139 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-10T03:16:14,211 DEBUG [RS:0;394d2222f724:38139 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/WALs/394d2222f724,38139,1731208572054 2024-11-10T03:16:14,213 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [394d2222f724,38139,1731208572054] 2024-11-10T03:16:14,237 INFO [RS:0;394d2222f724:38139 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-10T03:16:14,252 INFO [RS:0;394d2222f724:38139 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-10T03:16:14,257 INFO [RS:0;394d2222f724:38139 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-10T03:16:14,257 INFO [RS:0;394d2222f724:38139 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T03:16:14,259 INFO [RS:0;394d2222f724:38139 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-10T03:16:14,264 INFO [RS:0;394d2222f724:38139 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-10T03:16:14,265 INFO [RS:0;394d2222f724:38139 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-10T03:16:14,266 DEBUG [RS:0;394d2222f724:38139 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/394d2222f724:0, corePoolSize=1, maxPoolSize=1 2024-11-10T03:16:14,266 DEBUG [RS:0;394d2222f724:38139 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/394d2222f724:0, corePoolSize=1, maxPoolSize=1 2024-11-10T03:16:14,266 DEBUG [RS:0;394d2222f724:38139 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/394d2222f724:0, corePoolSize=1, maxPoolSize=1 2024-11-10T03:16:14,266 DEBUG [RS:0;394d2222f724:38139 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/394d2222f724:0, corePoolSize=1, maxPoolSize=1 2024-11-10T03:16:14,266 DEBUG [RS:0;394d2222f724:38139 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/394d2222f724:0, corePoolSize=1, maxPoolSize=1 2024-11-10T03:16:14,266 DEBUG [RS:0;394d2222f724:38139 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/394d2222f724:0, corePoolSize=2, maxPoolSize=2 2024-11-10T03:16:14,266 DEBUG [RS:0;394d2222f724:38139 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/394d2222f724:0, corePoolSize=1, maxPoolSize=1 2024-11-10T03:16:14,267 DEBUG [RS:0;394d2222f724:38139 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/394d2222f724:0, corePoolSize=1, maxPoolSize=1 2024-11-10T03:16:14,267 DEBUG [RS:0;394d2222f724:38139 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/394d2222f724:0, corePoolSize=1, maxPoolSize=1 2024-11-10T03:16:14,267 DEBUG [RS:0;394d2222f724:38139 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/394d2222f724:0, corePoolSize=1, maxPoolSize=1 2024-11-10T03:16:14,267 DEBUG [RS:0;394d2222f724:38139 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/394d2222f724:0, corePoolSize=1, maxPoolSize=1 2024-11-10T03:16:14,267 DEBUG [RS:0;394d2222f724:38139 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/394d2222f724:0, corePoolSize=1, maxPoolSize=1 2024-11-10T03:16:14,267 DEBUG [RS:0;394d2222f724:38139 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/394d2222f724:0, corePoolSize=3, maxPoolSize=3 2024-11-10T03:16:14,267 DEBUG [RS:0;394d2222f724:38139 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0, corePoolSize=3, maxPoolSize=3 2024-11-10T03:16:14,268 INFO [RS:0;394d2222f724:38139 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-10T03:16:14,268 INFO [RS:0;394d2222f724:38139 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-10T03:16:14,269 INFO [RS:0;394d2222f724:38139 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T03:16:14,269 INFO [RS:0;394d2222f724:38139 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-10T03:16:14,269 INFO [RS:0;394d2222f724:38139 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-10T03:16:14,269 INFO [RS:0;394d2222f724:38139 {}] hbase.ChoreService(168): Chore ScheduledChore name=394d2222f724,38139,1731208572054-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-10T03:16:14,290 INFO [RS:0;394d2222f724:38139 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-10T03:16:14,291 INFO [RS:0;394d2222f724:38139 {}] hbase.ChoreService(168): Chore ScheduledChore name=394d2222f724,38139,1731208572054-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T03:16:14,291 INFO [RS:0;394d2222f724:38139 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T03:16:14,292 INFO [RS:0;394d2222f724:38139 {}] regionserver.Replication(171): 394d2222f724,38139,1731208572054 started 2024-11-10T03:16:14,314 INFO [RS:0;394d2222f724:38139 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T03:16:14,314 INFO [RS:0;394d2222f724:38139 {}] regionserver.HRegionServer(1482): Serving as 394d2222f724,38139,1731208572054, RpcServer on 394d2222f724/172.17.0.3:38139, sessionid=0x10122be61aa0001 2024-11-10T03:16:14,315 DEBUG [RS:0;394d2222f724:38139 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-10T03:16:14,315 DEBUG [RS:0;394d2222f724:38139 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 394d2222f724,38139,1731208572054 2024-11-10T03:16:14,315 DEBUG [RS:0;394d2222f724:38139 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '394d2222f724,38139,1731208572054' 2024-11-10T03:16:14,315 DEBUG [RS:0;394d2222f724:38139 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-10T03:16:14,316 DEBUG [RS:0;394d2222f724:38139 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-10T03:16:14,317 DEBUG [RS:0;394d2222f724:38139 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-10T03:16:14,317 DEBUG [RS:0;394d2222f724:38139 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-10T03:16:14,318 DEBUG [RS:0;394d2222f724:38139 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 394d2222f724,38139,1731208572054 2024-11-10T03:16:14,318 DEBUG [RS:0;394d2222f724:38139 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '394d2222f724,38139,1731208572054' 2024-11-10T03:16:14,318 DEBUG [RS:0;394d2222f724:38139 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-10T03:16:14,319 DEBUG [RS:0;394d2222f724:38139 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-10T03:16:14,320 DEBUG [RS:0;394d2222f724:38139 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-10T03:16:14,320 INFO [RS:0;394d2222f724:38139 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-10T03:16:14,320 INFO [RS:0;394d2222f724:38139 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-10T03:16:14,427 INFO [RS:0;394d2222f724:38139 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-10T03:16:14,431 INFO [RS:0;394d2222f724:38139 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=394d2222f724%2C38139%2C1731208572054, suffix=, logDir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/WALs/394d2222f724,38139,1731208572054, archiveDir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/oldWALs, maxLogs=32 2024-11-10T03:16:14,446 DEBUG [RS:0;394d2222f724:38139 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/WALs/394d2222f724,38139,1731208572054/394d2222f724%2C38139%2C1731208572054.1731208574433, exclude list is [], retry=0 2024-11-10T03:16:14,450 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40501,DS-9059999b-6525-473a-934f-085b48080227,DISK] 2024-11-10T03:16:14,453 INFO [RS:0;394d2222f724:38139 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/WALs/394d2222f724,38139,1731208572054/394d2222f724%2C38139%2C1731208572054.1731208574433 2024-11-10T03:16:14,454 DEBUG [RS:0;394d2222f724:38139 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37975:37975)] 2024-11-10T03:16:14,460 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-10T03:16:14,460 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8 2024-11-10T03:16:14,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741833_1009 (size=32) 2024-11-10T03:16:14,872 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T03:16:14,875 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-10T03:16:14,878 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-10T03:16:14,879 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:14,879 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T03:16:14,880 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-10T03:16:14,882 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-10T03:16:14,882 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:14,884 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T03:16:14,884 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-10T03:16:14,887 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-10T03:16:14,887 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:14,888 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T03:16:14,888 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-10T03:16:14,891 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-10T03:16:14,891 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:14,892 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T03:16:14,892 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-10T03:16:14,894 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/hbase/meta/1588230740 2024-11-10T03:16:14,895 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/hbase/meta/1588230740 2024-11-10T03:16:14,898 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-10T03:16:14,898 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-10T03:16:14,899 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T03:16:14,902 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-10T03:16:14,907 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T03:16:14,908 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71814254, jitterRate=0.07011577486991882}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T03:16:14,910 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731208574873Initializing all the Stores at 1731208574875 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731208574875Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731208574875Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731208574875Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731208574875Cleaning up temporary data from old regions at 1731208574898 (+23 ms)Region opened successfully at 1731208574910 (+12 ms) 2024-11-10T03:16:14,911 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-10T03:16:14,911 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-10T03:16:14,911 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-10T03:16:14,911 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-10T03:16:14,911 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-10T03:16:14,913 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-10T03:16:14,913 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731208574911Disabling compacts and flushes for region at 1731208574911Disabling writes for close at 1731208574911Writing region close event to WAL at 1731208574912 (+1 ms)Closed at 1731208574913 (+1 ms) 2024-11-10T03:16:14,917 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-10T03:16:14,917 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-10T03:16:14,922 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-10T03:16:14,929 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-10T03:16:14,933 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-10T03:16:15,089 DEBUG [394d2222f724:37299 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-10T03:16:15,102 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=394d2222f724,38139,1731208572054 2024-11-10T03:16:15,108 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 394d2222f724,38139,1731208572054, state=OPENING 2024-11-10T03:16:15,178 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-10T03:16:15,188 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37299-0x10122be61aa0000, quorum=127.0.0.1:58617, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T03:16:15,188 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38139-0x10122be61aa0001, quorum=127.0.0.1:58617, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T03:16:15,190 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T03:16:15,190 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T03:16:15,192 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-10T03:16:15,194 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=394d2222f724,38139,1731208572054}] 2024-11-10T03:16:15,367 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-10T03:16:15,370 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42413, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-10T03:16:15,382 INFO [RS_OPEN_META-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-10T03:16:15,383 INFO [RS_OPEN_META-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-10T03:16:15,384 INFO [RS_OPEN_META-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-10T03:16:15,388 INFO [RS_OPEN_META-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=394d2222f724%2C38139%2C1731208572054.meta, suffix=.meta, logDir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/WALs/394d2222f724,38139,1731208572054, archiveDir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/oldWALs, maxLogs=32 2024-11-10T03:16:15,409 DEBUG [RS_OPEN_META-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/WALs/394d2222f724,38139,1731208572054/394d2222f724%2C38139%2C1731208572054.meta.1731208575391.meta, exclude list is [], retry=0 2024-11-10T03:16:15,413 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40501,DS-9059999b-6525-473a-934f-085b48080227,DISK] 2024-11-10T03:16:15,417 INFO [RS_OPEN_META-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/WALs/394d2222f724,38139,1731208572054/394d2222f724%2C38139%2C1731208572054.meta.1731208575391.meta 2024-11-10T03:16:15,418 DEBUG [RS_OPEN_META-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37975:37975)] 2024-11-10T03:16:15,418 DEBUG [RS_OPEN_META-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-10T03:16:15,420 DEBUG [RS_OPEN_META-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-10T03:16:15,423 DEBUG [RS_OPEN_META-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-10T03:16:15,429 INFO [RS_OPEN_META-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-10T03:16:15,434 DEBUG [RS_OPEN_META-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-10T03:16:15,435 DEBUG [RS_OPEN_META-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T03:16:15,435 DEBUG [RS_OPEN_META-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-10T03:16:15,435 DEBUG [RS_OPEN_META-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-10T03:16:15,440 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-10T03:16:15,442 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-10T03:16:15,442 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:15,444 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T03:16:15,444 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-10T03:16:15,447 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-10T03:16:15,447 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:15,448 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T03:16:15,449 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-10T03:16:15,451 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-10T03:16:15,451 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:15,452 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T03:16:15,452 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-10T03:16:15,455 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-10T03:16:15,455 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:15,456 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T03:16:15,457 DEBUG [RS_OPEN_META-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-10T03:16:15,459 DEBUG [RS_OPEN_META-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/hbase/meta/1588230740 2024-11-10T03:16:15,462 DEBUG [RS_OPEN_META-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/hbase/meta/1588230740 2024-11-10T03:16:15,465 DEBUG [RS_OPEN_META-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-10T03:16:15,465 DEBUG [RS_OPEN_META-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-10T03:16:15,466 DEBUG [RS_OPEN_META-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T03:16:15,469 DEBUG [RS_OPEN_META-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-10T03:16:15,470 INFO [RS_OPEN_META-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66173580, jitterRate=-0.013936817646026611}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T03:16:15,471 DEBUG [RS_OPEN_META-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-10T03:16:15,472 DEBUG [RS_OPEN_META-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731208575436Writing region info on filesystem at 1731208575436Initializing all the Stores at 1731208575438 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731208575439 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731208575440 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731208575440Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731208575440Cleaning up temporary data from old regions at 1731208575465 (+25 ms)Running coprocessor post-open hooks at 1731208575471 (+6 ms)Region opened successfully at 1731208575472 (+1 ms) 2024-11-10T03:16:15,480 INFO [RS_OPEN_META-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731208575360 2024-11-10T03:16:15,491 DEBUG [RS_OPEN_META-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-10T03:16:15,492 INFO [RS_OPEN_META-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-10T03:16:15,493 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=394d2222f724,38139,1731208572054 2024-11-10T03:16:15,495 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 394d2222f724,38139,1731208572054, state=OPEN 2024-11-10T03:16:15,538 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37299-0x10122be61aa0000, quorum=127.0.0.1:58617, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-10T03:16:15,538 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38139-0x10122be61aa0001, quorum=127.0.0.1:58617, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-10T03:16:15,538 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T03:16:15,538 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T03:16:15,539 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:15,545 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-10T03:16:15,546 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=394d2222f724,38139,1731208572054 in 345 msec 2024-11-10T03:16:15,552 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-10T03:16:15,553 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 625 msec 2024-11-10T03:16:15,554 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-10T03:16:15,554 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-10T03:16:15,573 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T03:16:15,574 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=394d2222f724,38139,1731208572054, seqNum=-1] 2024-11-10T03:16:15,593 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T03:16:15,596 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59733, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T03:16:15,618 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.6530 sec 2024-11-10T03:16:15,618 INFO [master/394d2222f724:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731208575618, completionTime=-1 2024-11-10T03:16:15,621 INFO [master/394d2222f724:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-10T03:16:15,621 DEBUG [master/394d2222f724:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-10T03:16:15,648 INFO [master/394d2222f724:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1 2024-11-10T03:16:15,649 INFO [master/394d2222f724:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731208635649 2024-11-10T03:16:15,649 INFO [master/394d2222f724:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731208695649 2024-11-10T03:16:15,649 INFO [master/394d2222f724:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 27 msec 2024-11-10T03:16:15,652 INFO [master/394d2222f724:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=394d2222f724,37299,1731208571229-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T03:16:15,652 INFO [master/394d2222f724:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=394d2222f724,37299,1731208571229-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T03:16:15,652 INFO [master/394d2222f724:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=394d2222f724,37299,1731208571229-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T03:16:15,654 INFO [master/394d2222f724:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-394d2222f724:37299, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T03:16:15,654 INFO [master/394d2222f724:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-10T03:16:15,655 INFO [master/394d2222f724:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-10T03:16:15,660 DEBUG [master/394d2222f724:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-10T03:16:15,680 INFO [master/394d2222f724:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.458sec 2024-11-10T03:16:15,681 INFO [master/394d2222f724:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-10T03:16:15,682 INFO [master/394d2222f724:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-10T03:16:15,683 INFO [master/394d2222f724:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-10T03:16:15,684 INFO [master/394d2222f724:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-10T03:16:15,684 INFO [master/394d2222f724:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-10T03:16:15,685 INFO [master/394d2222f724:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=394d2222f724,37299,1731208571229-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-10T03:16:15,686 INFO [master/394d2222f724:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=394d2222f724,37299,1731208571229-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-10T03:16:15,694 DEBUG [master/394d2222f724:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-10T03:16:15,695 INFO [master/394d2222f724:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-10T03:16:15,695 INFO [master/394d2222f724:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=394d2222f724,37299,1731208571229-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T03:16:15,763 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7935597f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:16:15,765 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-10T03:16:15,765 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-10T03:16:15,768 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 394d2222f724,37299,-1 for getting cluster id 2024-11-10T03:16:15,770 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T03:16:15,776 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f5c6727-65f7-4659-afe8-8e9a55664c78' 2024-11-10T03:16:15,779 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T03:16:15,779 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f5c6727-65f7-4659-afe8-8e9a55664c78" 2024-11-10T03:16:15,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70e3743c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:16:15,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [394d2222f724,37299,-1] 2024-11-10T03:16:15,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T03:16:15,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:16:15,787 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33074, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T03:16:15,789 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31132877, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:16:15,790 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T03:16:15,796 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=394d2222f724,38139,1731208572054, seqNum=-1] 2024-11-10T03:16:15,796 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T03:16:15,799 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51348, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T03:16:15,817 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=394d2222f724,37299,1731208571229 2024-11-10T03:16:15,827 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=686, ProcessCount=11, AvailableMemoryMB=3361 2024-11-10T03:16:15,834 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-10T03:16:15,839 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 394d2222f724,37299,1731208571229 2024-11-10T03:16:15,842 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@57db5d72 2024-11-10T03:16:15,843 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-10T03:16:15,846 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33090, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-10T03:16:15,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-10T03:16:15,868 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T03:16:15,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-10T03:16:15,883 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-10T03:16:15,885 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-10T03:16:15,885 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:15,889 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-10T03:16:15,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-10T03:16:15,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741835_1011 (size=1061) 2024-11-10T03:16:15,919 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8 2024-11-10T03:16:15,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741836_1012 (size=53) 2024-11-10T03:16:15,937 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T03:16:15,937 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 77ed36df592326523261438e65e5b722, disabling compactions & flushes 2024-11-10T03:16:15,937 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:15,937 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:15,937 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. after waiting 0 ms 2024-11-10T03:16:15,937 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:15,937 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:15,938 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 77ed36df592326523261438e65e5b722: Waiting for close lock at 1731208575937Disabling compacts and flushes for region at 1731208575937Disabling writes for close at 1731208575937Writing region close event to WAL at 1731208575937Closed at 1731208575937 2024-11-10T03:16:15,940 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-10T03:16:15,947 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731208575941"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731208575941"}]},"ts":"1731208575941"} 2024-11-10T03:16:15,953 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-10T03:16:15,956 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-10T03:16:15,959 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731208575956"}]},"ts":"1731208575956"} 2024-11-10T03:16:15,970 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-10T03:16:15,973 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=77ed36df592326523261438e65e5b722, ASSIGN}] 2024-11-10T03:16:15,977 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=77ed36df592326523261438e65e5b722, ASSIGN 2024-11-10T03:16:15,980 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=77ed36df592326523261438e65e5b722, ASSIGN; state=OFFLINE, location=394d2222f724,38139,1731208572054; forceNewPlan=false, retain=false 2024-11-10T03:16:16,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-10T03:16:16,132 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=77ed36df592326523261438e65e5b722, regionState=OPENING, regionLocation=394d2222f724,38139,1731208572054 2024-11-10T03:16:16,138 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=77ed36df592326523261438e65e5b722, ASSIGN because future has completed 2024-11-10T03:16:16,139 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054}] 2024-11-10T03:16:16,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-10T03:16:16,302 INFO [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:16,303 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} 2024-11-10T03:16:16,303 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 77ed36df592326523261438e65e5b722 2024-11-10T03:16:16,303 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T03:16:16,304 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 77ed36df592326523261438e65e5b722 2024-11-10T03:16:16,304 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 77ed36df592326523261438e65e5b722 2024-11-10T03:16:16,306 INFO [StoreOpener-77ed36df592326523261438e65e5b722-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 77ed36df592326523261438e65e5b722 2024-11-10T03:16:16,311 INFO [StoreOpener-77ed36df592326523261438e65e5b722-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T03:16:16,312 INFO [StoreOpener-77ed36df592326523261438e65e5b722-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 77ed36df592326523261438e65e5b722 columnFamilyName A 2024-11-10T03:16:16,312 DEBUG [StoreOpener-77ed36df592326523261438e65e5b722-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:16,313 INFO [StoreOpener-77ed36df592326523261438e65e5b722-1 {}] regionserver.HStore(327): Store=77ed36df592326523261438e65e5b722/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T03:16:16,314 INFO [StoreOpener-77ed36df592326523261438e65e5b722-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 77ed36df592326523261438e65e5b722 2024-11-10T03:16:16,316 INFO [StoreOpener-77ed36df592326523261438e65e5b722-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T03:16:16,317 INFO [StoreOpener-77ed36df592326523261438e65e5b722-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 77ed36df592326523261438e65e5b722 columnFamilyName B 2024-11-10T03:16:16,317 DEBUG [StoreOpener-77ed36df592326523261438e65e5b722-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:16,318 INFO [StoreOpener-77ed36df592326523261438e65e5b722-1 {}] regionserver.HStore(327): Store=77ed36df592326523261438e65e5b722/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T03:16:16,318 INFO [StoreOpener-77ed36df592326523261438e65e5b722-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 77ed36df592326523261438e65e5b722 2024-11-10T03:16:16,320 INFO [StoreOpener-77ed36df592326523261438e65e5b722-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T03:16:16,321 INFO [StoreOpener-77ed36df592326523261438e65e5b722-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 77ed36df592326523261438e65e5b722 columnFamilyName C 2024-11-10T03:16:16,321 DEBUG [StoreOpener-77ed36df592326523261438e65e5b722-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:16,322 INFO [StoreOpener-77ed36df592326523261438e65e5b722-1 {}] regionserver.HStore(327): Store=77ed36df592326523261438e65e5b722/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T03:16:16,322 INFO [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:16,322 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 77ed36df592326523261438e65e5b722 2024-11-10T03:16:16,324 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722 2024-11-10T03:16:16,325 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722 2024-11-10T03:16:16,327 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 77ed36df592326523261438e65e5b722 2024-11-10T03:16:16,327 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 77ed36df592326523261438e65e5b722 2024-11-10T03:16:16,328 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T03:16:16,330 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 77ed36df592326523261438e65e5b722 2024-11-10T03:16:16,334 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T03:16:16,335 INFO [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 77ed36df592326523261438e65e5b722; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69636871, jitterRate=0.03767023980617523}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T03:16:16,335 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 77ed36df592326523261438e65e5b722 2024-11-10T03:16:16,336 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 77ed36df592326523261438e65e5b722: Running coprocessor pre-open hook at 1731208576304Writing region info on filesystem at 1731208576304Initializing all the Stores at 1731208576306 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731208576306Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731208576306Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731208576306Cleaning up temporary data from old regions at 1731208576327 (+21 ms)Running coprocessor post-open hooks at 1731208576335 (+8 ms)Region opened successfully at 1731208576336 (+1 ms) 2024-11-10T03:16:16,338 INFO [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., pid=6, masterSystemTime=1731208576296 2024-11-10T03:16:16,342 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:16,342 INFO [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:16,343 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=77ed36df592326523261438e65e5b722, regionState=OPEN, openSeqNum=2, regionLocation=394d2222f724,38139,1731208572054 2024-11-10T03:16:16,348 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 because future has completed 2024-11-10T03:16:16,359 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-10T03:16:16,360 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 in 212 msec 2024-11-10T03:16:16,366 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-10T03:16:16,366 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=77ed36df592326523261438e65e5b722, ASSIGN in 387 msec 2024-11-10T03:16:16,369 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-10T03:16:16,369 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731208576369"}]},"ts":"1731208576369"} 2024-11-10T03:16:16,373 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-10T03:16:16,376 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-10T03:16:16,383 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 503 msec 2024-11-10T03:16:16,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-10T03:16:16,536 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-10T03:16:16,545 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24cd370b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:16:16,545 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 394d2222f724,37299,-1 for getting cluster id 2024-11-10T03:16:16,545 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T03:16:16,547 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f5c6727-65f7-4659-afe8-8e9a55664c78' 2024-11-10T03:16:16,547 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T03:16:16,548 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f5c6727-65f7-4659-afe8-8e9a55664c78" 2024-11-10T03:16:16,548 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a220135, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:16:16,548 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [394d2222f724,37299,-1] 2024-11-10T03:16:16,548 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T03:16:16,549 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:16:16,551 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33112, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T03:16:16,553 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@322a9c3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:16:16,553 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T03:16:16,555 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=394d2222f724,38139,1731208572054, seqNum=-1] 2024-11-10T03:16:16,556 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T03:16:16,558 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51350, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T03:16:16,561 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-10T03:16:16,563 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 394d2222f724,37299,1731208571229 2024-11-10T03:16:16,563 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@63481454 2024-11-10T03:16:16,566 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-10T03:16:16,569 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33114, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-10T03:16:16,573 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a25f29a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:16:16,573 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 394d2222f724,37299,-1 for getting cluster id 2024-11-10T03:16:16,574 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T03:16:16,575 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f5c6727-65f7-4659-afe8-8e9a55664c78' 2024-11-10T03:16:16,576 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T03:16:16,576 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f5c6727-65f7-4659-afe8-8e9a55664c78" 2024-11-10T03:16:16,577 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b130fb1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:16:16,577 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [394d2222f724,37299,-1] 2024-11-10T03:16:16,577 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T03:16:16,578 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:16:16,579 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33128, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T03:16:16,580 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d1d7645, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:16:16,581 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8953fc5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:16:16,581 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 394d2222f724,37299,-1 for getting cluster id 2024-11-10T03:16:16,581 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T03:16:16,583 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f5c6727-65f7-4659-afe8-8e9a55664c78' 2024-11-10T03:16:16,583 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T03:16:16,584 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f5c6727-65f7-4659-afe8-8e9a55664c78" 2024-11-10T03:16:16,584 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27ff6f12, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:16:16,584 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [394d2222f724,37299,-1] 2024-11-10T03:16:16,584 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T03:16:16,585 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:16:16,586 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33160, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T03:16:16,587 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10bc6ecb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:16:16,588 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75df9ef2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:16:16,588 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 394d2222f724,37299,-1 for getting cluster id 2024-11-10T03:16:16,588 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T03:16:16,590 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f5c6727-65f7-4659-afe8-8e9a55664c78' 2024-11-10T03:16:16,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T03:16:16,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f5c6727-65f7-4659-afe8-8e9a55664c78" 2024-11-10T03:16:16,591 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ff1f1f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:16:16,591 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [394d2222f724,37299,-1] 2024-11-10T03:16:16,592 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T03:16:16,592 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:16:16,593 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33178, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T03:16:16,595 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b37f010, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:16:16,596 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5731bbe2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:16:16,596 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 394d2222f724,37299,-1 for getting cluster id 2024-11-10T03:16:16,596 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T03:16:16,597 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f5c6727-65f7-4659-afe8-8e9a55664c78' 2024-11-10T03:16:16,598 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T03:16:16,598 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f5c6727-65f7-4659-afe8-8e9a55664c78" 2024-11-10T03:16:16,598 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38788dd6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:16:16,598 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [394d2222f724,37299,-1] 2024-11-10T03:16:16,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T03:16:16,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:16:16,601 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33196, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T03:16:16,602 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35818969, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:16:16,603 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c885828, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:16:16,603 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 394d2222f724,37299,-1 for getting cluster id 2024-11-10T03:16:16,603 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T03:16:16,604 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f5c6727-65f7-4659-afe8-8e9a55664c78' 2024-11-10T03:16:16,604 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T03:16:16,605 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f5c6727-65f7-4659-afe8-8e9a55664c78" 2024-11-10T03:16:16,605 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b688ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:16:16,605 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [394d2222f724,37299,-1] 2024-11-10T03:16:16,605 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T03:16:16,605 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:16:16,607 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33218, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T03:16:16,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@344fa400, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:16:16,609 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28287983, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:16:16,609 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 394d2222f724,37299,-1 for getting cluster id 2024-11-10T03:16:16,609 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T03:16:16,610 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f5c6727-65f7-4659-afe8-8e9a55664c78' 2024-11-10T03:16:16,611 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T03:16:16,611 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f5c6727-65f7-4659-afe8-8e9a55664c78" 2024-11-10T03:16:16,612 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@eac2906, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:16:16,612 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [394d2222f724,37299,-1] 2024-11-10T03:16:16,612 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T03:16:16,612 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:16:16,613 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33232, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T03:16:16,615 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@287272c0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:16:16,615 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c04eae4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:16:16,615 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 394d2222f724,37299,-1 for getting cluster id 2024-11-10T03:16:16,616 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T03:16:16,617 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f5c6727-65f7-4659-afe8-8e9a55664c78' 2024-11-10T03:16:16,617 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T03:16:16,617 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f5c6727-65f7-4659-afe8-8e9a55664c78" 2024-11-10T03:16:16,617 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@788043de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:16:16,618 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [394d2222f724,37299,-1] 2024-11-10T03:16:16,618 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T03:16:16,618 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:16:16,619 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33254, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T03:16:16,620 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@421973e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:16:16,621 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d75e95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:16:16,621 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 394d2222f724,37299,-1 for getting cluster id 2024-11-10T03:16:16,622 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T03:16:16,623 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f5c6727-65f7-4659-afe8-8e9a55664c78' 2024-11-10T03:16:16,623 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T03:16:16,623 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f5c6727-65f7-4659-afe8-8e9a55664c78" 2024-11-10T03:16:16,624 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78238dbc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:16:16,624 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [394d2222f724,37299,-1] 2024-11-10T03:16:16,624 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T03:16:16,624 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:16:16,625 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33266, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T03:16:16,626 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76c8af47, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:16:16,627 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c9d7e50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:16:16,627 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 394d2222f724,37299,-1 for getting cluster id 2024-11-10T03:16:16,627 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T03:16:16,628 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f5c6727-65f7-4659-afe8-8e9a55664c78' 2024-11-10T03:16:16,629 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T03:16:16,629 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f5c6727-65f7-4659-afe8-8e9a55664c78" 2024-11-10T03:16:16,629 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13c29b6b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:16:16,629 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [394d2222f724,37299,-1] 2024-11-10T03:16:16,629 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T03:16:16,630 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:16:16,631 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33284, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T03:16:16,632 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78aaa9c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:16:16,638 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T03:16:16,638 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T03:16:16,639 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T03:16:16,639 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T03:16:16,640 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T03:16:16,641 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=394d2222f724,38139,1731208572054, seqNum=-1] 2024-11-10T03:16:16,641 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T03:16:16,641 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T03:16:16,642 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=394d2222f724,38139,1731208572054, seqNum=-1] 2024-11-10T03:16:16,642 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=394d2222f724,38139,1731208572054, seqNum=-1] 2024-11-10T03:16:16,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=394d2222f724,38139,1731208572054, seqNum=-1] 2024-11-10T03:16:16,643 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T03:16:16,643 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T03:16:16,643 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T03:16:16,643 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=394d2222f724,38139,1731208572054, seqNum=-1] 2024-11-10T03:16:16,644 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T03:16:16,644 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51352, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T03:16:16,645 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51378, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T03:16:16,645 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51370, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T03:16:16,645 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51380, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T03:16:16,646 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51354, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T03:16:16,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-10T03:16:16,649 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T03:16:16,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-10T03:16:16,652 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T03:16:16,653 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2] 2024-11-10T03:16:16,653 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T03:16:16,653 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T03:16:16,654 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2] 2024-11-10T03:16:16,655 DEBUG [Thread-158 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=394d2222f724,38139,1731208572054, seqNum=-1] 2024-11-10T03:16:16,655 DEBUG [Thread-158 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T03:16:16,657 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2] 2024-11-10T03:16:16,659 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2] 2024-11-10T03:16:16,659 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T03:16:16,659 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51396, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T03:16:16,659 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T03:16:16,660 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=394d2222f724,38139,1731208572054, seqNum=-1] 2024-11-10T03:16:16,661 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=394d2222f724,38139,1731208572054, seqNum=-1] 2024-11-10T03:16:16,661 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T03:16:16,661 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T03:16:16,661 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2] 2024-11-10T03:16:16,662 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51400, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T03:16:16,665 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2] 2024-11-10T03:16:16,666 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51402, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T03:16:16,671 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2] 2024-11-10T03:16:16,674 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T03:16:16,676 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=394d2222f724,38139,1731208572054, seqNum=-1] 2024-11-10T03:16:16,677 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T03:16:16,677 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2] 2024-11-10T03:16:16,679 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51428, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T03:16:16,683 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2] 2024-11-10T03:16:16,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on 77ed36df592326523261438e65e5b722 2024-11-10T03:16:16,744 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 77ed36df592326523261438e65e5b722 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-10T03:16:16,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-10T03:16:16,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77ed36df592326523261438e65e5b722, store=A 2024-11-10T03:16:16,756 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:16:16,756 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77ed36df592326523261438e65e5b722, store=B 2024-11-10T03:16:16,757 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:16:16,757 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77ed36df592326523261438e65e5b722, store=C 2024-11-10T03:16:16,757 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:16:16,839 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-10T03:16:16,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:16,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. as already flushing 2024-11-10T03:16:16,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:16,854 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:16,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:16,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:16,939 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/A/67b942361abf4722b46bb67189b79d92 is 50, key is test_row_0/A:col10/1731208576729/Put/seqid=0 2024-11-10T03:16:16,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:16,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51402 deadline: 1731208636890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:16,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:16,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51380 deadline: 1731208636903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:16,955 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:16,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:16,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:16,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:16,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:16,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:16,960 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731208576866, max timestamp=1731208576906Num uniques 859; 2024-11-10T03:16:16,962 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T03:16:16,964 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731208576866, max timestamp=1731208576902Num uniques 859; 2024-11-10T03:16:16,965 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-10T03:16:16,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-10T03:16:16,986 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731208576866, max timestamp=1731208576906Num uniques 859; 2024-11-10T03:16:16,986 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T03:16:16,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741837_1013 (size=14341) 2024-11-10T03:16:16,997 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/A/67b942361abf4722b46bb67189b79d92 2024-11-10T03:16:17,029 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-10T03:16:17,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:17,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. as already flushing 2024-11-10T03:16:17,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:17,043 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:17,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:17,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:17,115 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:17,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51402 deadline: 1731208637082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:17,117 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:17,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51354 deadline: 1731208637086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:17,119 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:17,119 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:17,119 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:17,121 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:17,121 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:17,121 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:17,122 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:17,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51370 deadline: 1731208637098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:17,123 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:17,123 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:17,123 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:17,130 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:17,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51380 deadline: 1731208637093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:17,132 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:17,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51378 deadline: 1731208637114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:17,133 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:17,133 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:17,134 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:17,133 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:17,134 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:17,134 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:17,175 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/B/1df2dd6e105343ea8b21dffafc8b9927 is 50, key is test_row_0/B:col10/1731208576729/Put/seqid=0 2024-11-10T03:16:17,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741838_1014 (size=12001) 2024-11-10T03:16:17,197 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/B/1df2dd6e105343ea8b21dffafc8b9927 2024-11-10T03:16:17,206 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-10T03:16:17,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:17,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. as already flushing 2024-11-10T03:16:17,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:17,216 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:17,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:17,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:17,234 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:17,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51354 deadline: 1731208637227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:17,236 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:17,236 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:17,236 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:17,238 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:17,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51402 deadline: 1731208637235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:17,240 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:17,240 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:17,240 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:17,243 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:17,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51370 deadline: 1731208637237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:17,245 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:17,245 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:17,245 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:17,246 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:17,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51378 deadline: 1731208637245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:17,248 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:17,248 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:17,248 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:17,251 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:17,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51380 deadline: 1731208637247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:17,252 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:17,253 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:17,253 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:17,280 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/C/f79000458af44e5c9770b1f5f651e383 is 50, key is test_row_0/C:col10/1731208576729/Put/seqid=0 2024-11-10T03:16:17,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-10T03:16:17,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741839_1015 (size=12001) 2024-11-10T03:16:17,327 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/C/f79000458af44e5c9770b1f5f651e383 2024-11-10T03:16:17,345 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/A/67b942361abf4722b46bb67189b79d92 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/67b942361abf4722b46bb67189b79d92 2024-11-10T03:16:17,363 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/67b942361abf4722b46bb67189b79d92, entries=200, sequenceid=12, filesize=14.0 K 2024-11-10T03:16:17,370 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/B/1df2dd6e105343ea8b21dffafc8b9927 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/1df2dd6e105343ea8b21dffafc8b9927 2024-11-10T03:16:17,373 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-10T03:16:17,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:17,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. as already flushing 2024-11-10T03:16:17,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:17,374 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:17,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:17,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:17,391 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/1df2dd6e105343ea8b21dffafc8b9927, entries=150, sequenceid=12, filesize=11.7 K 2024-11-10T03:16:17,399 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/C/f79000458af44e5c9770b1f5f651e383 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/f79000458af44e5c9770b1f5f651e383 2024-11-10T03:16:17,424 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/f79000458af44e5c9770b1f5f651e383, entries=150, sequenceid=12, filesize=11.7 K 2024-11-10T03:16:17,438 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.08 KB/130125 for 77ed36df592326523261438e65e5b722 in 692ms, sequenceid=12, compaction requested=false 2024-11-10T03:16:17,442 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-10T03:16:17,446 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 77ed36df592326523261438e65e5b722: 2024-11-10T03:16:17,457 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 77ed36df592326523261438e65e5b722 3/3 column families, dataSize=133.78 KB heapSize=384.07 KB 2024-11-10T03:16:17,457 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77ed36df592326523261438e65e5b722, store=A 2024-11-10T03:16:17,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on 77ed36df592326523261438e65e5b722 2024-11-10T03:16:17,460 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T03:16:17,460 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77ed36df592326523261438e65e5b722, store=B 2024-11-10T03:16:17,460 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T03:16:17,460 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77ed36df592326523261438e65e5b722, store=C 2024-11-10T03:16:17,460 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T03:16:17,494 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/A/c5f104db49ae4b59ba400cb62edc6433 is 50, key is test_row_0/A:col10/1731208577453/Put/seqid=0 2024-11-10T03:16:17,518 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:17,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51378 deadline: 1731208637492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:17,520 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:17,520 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:17,520 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:17,521 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:17,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51354 deadline: 1731208637497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:17,522 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:17,522 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:17,522 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:17,522 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:17,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51380 deadline: 1731208637499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:17,523 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:17,524 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:17,524 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:17,524 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:17,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51402 deadline: 1731208637516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:17,531 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:17,531 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:17,531 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:17,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741840_1016 (size=12001) 2024-11-10T03:16:17,545 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.33 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/A/c5f104db49ae4b59ba400cb62edc6433 2024-11-10T03:16:17,546 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:17,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51370 deadline: 1731208637520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:17,550 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-10T03:16:17,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:17,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. as already flushing 2024-11-10T03:16:17,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:17,551 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:17,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:17,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:17,556 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:17,556 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:17,557 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:17,582 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/B/a50baa4568a24aceb7aafe83deab7ae3 is 50, key is test_row_0/B:col10/1731208577453/Put/seqid=0 2024-11-10T03:16:17,629 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:17,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51378 deadline: 1731208637624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:17,633 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:17,633 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:17,633 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:17,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741841_1017 (size=12001) 2024-11-10T03:16:17,650 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/B/a50baa4568a24aceb7aafe83deab7ae3 2024-11-10T03:16:17,651 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:17,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51354 deadline: 1731208637635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:17,653 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:17,653 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:17,653 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:17,654 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:17,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51380 deadline: 1731208637635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:17,656 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:17,656 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:17,656 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:17,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:17,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51402 deadline: 1731208637636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:17,659 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:17,659 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:17,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:17,668 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:17,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51370 deadline: 1731208637665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:17,670 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:17,670 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:17,670 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:17,702 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/C/a70057a05b2547d8a434878573a74db6 is 50, key is test_row_0/C:col10/1731208577453/Put/seqid=0 2024-11-10T03:16:17,707 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-10T03:16:17,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:17,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. as already flushing 2024-11-10T03:16:17,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:17,708 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:17,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:17,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:17,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741842_1018 (size=12001) 2024-11-10T03:16:17,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-10T03:16:17,857 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:17,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51378 deadline: 1731208637843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:17,858 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:17,858 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:17,859 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:17,870 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-10T03:16:17,870 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:17,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:17,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51402 deadline: 1731208637865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:17,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. as already flushing 2024-11-10T03:16:17,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:17,871 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:17,871 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:17,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:17,871 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:17,872 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:17,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:17,876 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:17,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51354 deadline: 1731208637869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:17,878 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:17,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51380 deadline: 1731208637869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:17,881 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:17,881 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:17,881 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:17,882 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:17,882 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:17,882 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:17,889 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:17,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51370 deadline: 1731208637877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:17,890 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:17,891 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:17,891 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:18,027 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-10T03:16:18,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:18,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. as already flushing 2024-11-10T03:16:18,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:18,028 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:18,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:18,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:18,166 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/C/a70057a05b2547d8a434878573a74db6 2024-11-10T03:16:18,185 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:18,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51378 deadline: 1731208638176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:18,187 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:18,187 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:18,187 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:18,190 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:18,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51402 deadline: 1731208638184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:18,192 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:18,192 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:18,192 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:18,196 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/A/c5f104db49ae4b59ba400cb62edc6433 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/c5f104db49ae4b59ba400cb62edc6433 2024-11-10T03:16:18,201 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-10T03:16:18,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:18,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. as already flushing 2024-11-10T03:16:18,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:18,201 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:18,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:18,202 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:18,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51354 deadline: 1731208638194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:18,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:18,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:18,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51380 deadline: 1731208638194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:18,208 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:18,208 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:18,208 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:18,208 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:18,208 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:18,208 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:18,216 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:18,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51370 deadline: 1731208638206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:18,219 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:18,220 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:18,220 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:18,222 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/c5f104db49ae4b59ba400cb62edc6433, entries=150, sequenceid=49, filesize=11.7 K 2024-11-10T03:16:18,235 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/B/a50baa4568a24aceb7aafe83deab7ae3 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/a50baa4568a24aceb7aafe83deab7ae3 2024-11-10T03:16:18,273 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/a50baa4568a24aceb7aafe83deab7ae3, entries=150, sequenceid=49, filesize=11.7 K 2024-11-10T03:16:18,276 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/C/a70057a05b2547d8a434878573a74db6 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/a70057a05b2547d8a434878573a74db6 2024-11-10T03:16:18,314 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/a70057a05b2547d8a434878573a74db6, entries=150, sequenceid=49, filesize=11.7 K 2024-11-10T03:16:18,319 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~133.78 KB/136995, heapSize ~384.02 KB/393240, currentSize=46.96 KB/48090 for 77ed36df592326523261438e65e5b722 in 862ms, sequenceid=49, compaction requested=false 2024-11-10T03:16:18,319 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 77ed36df592326523261438e65e5b722: 2024-11-10T03:16:18,367 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-10T03:16:18,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:18,368 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 77ed36df592326523261438e65e5b722 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-10T03:16:18,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77ed36df592326523261438e65e5b722, store=A 2024-11-10T03:16:18,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:16:18,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77ed36df592326523261438e65e5b722, store=B 2024-11-10T03:16:18,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:16:18,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77ed36df592326523261438e65e5b722, store=C 2024-11-10T03:16:18,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:16:18,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/A/f4be510c6299423b9337909a8a491132 is 50, key is test_row_0/A:col10/1731208577492/Put/seqid=0 2024-11-10T03:16:18,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741843_1019 (size=12001) 2024-11-10T03:16:18,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on 77ed36df592326523261438e65e5b722 2024-11-10T03:16:18,709 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. as already flushing 2024-11-10T03:16:18,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-10T03:16:18,844 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/A/f4be510c6299423b9337909a8a491132 2024-11-10T03:16:18,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/B/460d9c2f418041a59e01afada5998cec is 50, key is test_row_0/B:col10/1731208577492/Put/seqid=0 2024-11-10T03:16:18,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741844_1020 (size=12001) 2024-11-10T03:16:18,929 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/B/460d9c2f418041a59e01afada5998cec 2024-11-10T03:16:18,935 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731208578774, max timestamp=1731208578922Num uniques 859; 2024-11-10T03:16:18,936 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T03:16:18,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/C/c857d14211d94a209f81c919a922c5e5 is 50, key is test_row_0/C:col10/1731208577492/Put/seqid=0 2024-11-10T03:16:18,958 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731208578774, max timestamp=1731208578922Num uniques 859; 2024-11-10T03:16:18,958 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T03:16:18,974 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731208578774, max timestamp=1731208578922Num uniques 859; 2024-11-10T03:16:18,988 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-10T03:16:18,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741845_1021 (size=12001) 2024-11-10T03:16:19,045 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:19,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51354 deadline: 1731208639027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:19,047 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:19,047 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:19,047 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:19,047 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:19,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51370 deadline: 1731208639038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:19,048 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:19,049 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:19,049 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:19,052 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731208579028, max timestamp=1731208579043Num uniques 1018; 2024-11-10T03:16:19,053 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-10T03:16:19,064 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731208579028, max timestamp=1731208579043Num uniques 1018; 2024-11-10T03:16:19,064 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-10T03:16:19,084 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731208579028, max timestamp=1731208579043Num uniques 1009; 2024-11-10T03:16:19,085 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46213, new segment data size=6870, suffix heap size=132904,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=150 2024-11-10T03:16:19,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:19,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51354 deadline: 1731208639154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:19,166 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:19,166 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:19,166 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:19,168 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:19,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51380 deadline: 1731208639162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:19,169 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:19,169 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:19,169 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:19,169 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:19,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51402 deadline: 1731208639162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:19,170 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:19,170 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:19,170 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:19,170 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731208579113, max timestamp=1731208579163Num uniques 1009; 2024-11-10T03:16:19,170 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=10392, suffix heap size=132904,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=227 2024-11-10T03:16:19,171 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731208579113, max timestamp=1731208579163Num uniques 1027; 2024-11-10T03:16:19,171 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-11-10T03:16:19,178 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731208579157, max timestamp=1731208579163Num uniques 1027; 2024-11-10T03:16:19,178 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-10T03:16:19,268 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:19,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51370 deadline: 1731208639262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:19,269 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:19,269 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:19,270 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:19,270 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:19,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51378 deadline: 1731208639269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:19,271 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:19,271 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:19,272 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:19,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:19,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51402 deadline: 1731208639273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:19,287 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:19,287 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:19,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:19,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:19,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51380 deadline: 1731208639274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:19,291 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:19,291 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:19,291 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:19,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:19,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51354 deadline: 1731208639373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:19,380 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:19,380 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:19,380 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:19,380 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:19,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51370 deadline: 1731208639374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:19,381 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:19,381 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:19,381 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:19,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:19,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51378 deadline: 1731208639389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:19,391 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:19,391 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:19,391 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:19,399 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/C/c857d14211d94a209f81c919a922c5e5 2024-11-10T03:16:19,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/A/f4be510c6299423b9337909a8a491132 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/f4be510c6299423b9337909a8a491132 2024-11-10T03:16:19,433 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/f4be510c6299423b9337909a8a491132, entries=150, sequenceid=59, filesize=11.7 K 2024-11-10T03:16:19,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/B/460d9c2f418041a59e01afada5998cec as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/460d9c2f418041a59e01afada5998cec 2024-11-10T03:16:19,449 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/460d9c2f418041a59e01afada5998cec, entries=150, sequenceid=59, filesize=11.7 K 2024-11-10T03:16:19,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/C/c857d14211d94a209f81c919a922c5e5 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/c857d14211d94a209f81c919a922c5e5 2024-11-10T03:16:19,467 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/c857d14211d94a209f81c919a922c5e5, entries=150, sequenceid=59, filesize=11.7 K 2024-11-10T03:16:19,471 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=131.94 KB/135110 for 77ed36df592326523261438e65e5b722 in 1103ms, sequenceid=59, compaction requested=true 2024-11-10T03:16:19,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 77ed36df592326523261438e65e5b722: 2024-11-10T03:16:19,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:19,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-10T03:16:19,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-10T03:16:19,478 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-10T03:16:19,479 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8210 sec 2024-11-10T03:16:19,484 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 2.8390 sec 2024-11-10T03:16:19,498 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 77ed36df592326523261438e65e5b722 3/3 column families, dataSize=138.65 KB heapSize=393.45 KB 2024-11-10T03:16:19,499 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77ed36df592326523261438e65e5b722, store=A 2024-11-10T03:16:19,499 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T03:16:19,499 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77ed36df592326523261438e65e5b722, store=B 2024-11-10T03:16:19,499 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T03:16:19,499 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77ed36df592326523261438e65e5b722, store=C 2024-11-10T03:16:19,499 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T03:16:19,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on 77ed36df592326523261438e65e5b722 2024-11-10T03:16:19,520 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/A/d56b8bbae5724177ba16577be40d3aea is 50, key is test_row_0/A:col10/1731208579239/Put/seqid=0 2024-11-10T03:16:19,571 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:19,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51402 deadline: 1731208639561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:19,572 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:19,576 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:19,576 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:19,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:19,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51380 deadline: 1731208639566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:19,584 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:19,584 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:19,584 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:19,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741846_1022 (size=14391) 2024-11-10T03:16:19,587 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/A/d56b8bbae5724177ba16577be40d3aea 2024-11-10T03:16:19,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:19,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51378 deadline: 1731208639594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:19,602 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:19,602 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:19,602 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:19,602 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:19,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51370 deadline: 1731208639595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:19,603 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:19,603 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:19,603 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:19,616 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/B/7645396c537e44168393ed1b88f13433 is 50, key is test_row_0/B:col10/1731208579239/Put/seqid=0 2024-11-10T03:16:19,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741847_1023 (size=12051) 2024-11-10T03:16:19,679 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/B/7645396c537e44168393ed1b88f13433 2024-11-10T03:16:19,687 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:19,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51402 deadline: 1731208639683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:19,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:19,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:19,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:19,703 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:19,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51354 deadline: 1731208639693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:19,705 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:19,705 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:19,705 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:19,706 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:19,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51380 deadline: 1731208639694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:19,707 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:19,707 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:19,707 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:19,715 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/C/9636f7c4fef04615b25eac6614ad11b2 is 50, key is test_row_0/C:col10/1731208579239/Put/seqid=0 2024-11-10T03:16:19,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741848_1024 (size=12051) 2024-11-10T03:16:19,755 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/C/9636f7c4fef04615b25eac6614ad11b2 2024-11-10T03:16:19,769 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/A/d56b8bbae5724177ba16577be40d3aea as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/d56b8bbae5724177ba16577be40d3aea 2024-11-10T03:16:19,788 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/d56b8bbae5724177ba16577be40d3aea, entries=200, sequenceid=130, filesize=14.1 K 2024-11-10T03:16:19,791 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/B/7645396c537e44168393ed1b88f13433 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/7645396c537e44168393ed1b88f13433 2024-11-10T03:16:19,813 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/7645396c537e44168393ed1b88f13433, entries=150, sequenceid=130, filesize=11.8 K 2024-11-10T03:16:19,816 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/C/9636f7c4fef04615b25eac6614ad11b2 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/9636f7c4fef04615b25eac6614ad11b2 2024-11-10T03:16:19,839 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/9636f7c4fef04615b25eac6614ad11b2, entries=150, sequenceid=130, filesize=11.8 K 2024-11-10T03:16:19,844 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~138.65 KB/141980, heapSize ~393.41 KB/402848, currentSize=40.25 KB/41220 for 77ed36df592326523261438e65e5b722 in 346ms, sequenceid=130, compaction requested=true 2024-11-10T03:16:19,844 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 77ed36df592326523261438e65e5b722: 2024-11-10T03:16:19,846 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 77ed36df592326523261438e65e5b722:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T03:16:19,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:16:19,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 77ed36df592326523261438e65e5b722:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T03:16:19,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:16:19,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 77ed36df592326523261438e65e5b722:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T03:16:19,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T03:16:19,851 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T03:16:19,851 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T03:16:19,859 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48054 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T03:16:19,861 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1541): 77ed36df592326523261438e65e5b722/B is initiating minor compaction (all files) 2024-11-10T03:16:19,862 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 77ed36df592326523261438e65e5b722/B in TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:19,862 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/1df2dd6e105343ea8b21dffafc8b9927, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/a50baa4568a24aceb7aafe83deab7ae3, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/460d9c2f418041a59e01afada5998cec, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/7645396c537e44168393ed1b88f13433] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp, totalSize=46.9 K 2024-11-10T03:16:19,864 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 1df2dd6e105343ea8b21dffafc8b9927, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731208576722 2024-11-10T03:16:19,866 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting a50baa4568a24aceb7aafe83deab7ae3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731208577092 2024-11-10T03:16:19,867 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 460d9c2f418041a59e01afada5998cec, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731208577486 2024-11-10T03:16:19,868 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 7645396c537e44168393ed1b88f13433, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731208579201 2024-11-10T03:16:19,873 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52734 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T03:16:19,874 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1541): 77ed36df592326523261438e65e5b722/A is initiating minor compaction (all files) 2024-11-10T03:16:19,874 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 77ed36df592326523261438e65e5b722/A in TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:19,874 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/67b942361abf4722b46bb67189b79d92, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/c5f104db49ae4b59ba400cb62edc6433, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/f4be510c6299423b9337909a8a491132, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/d56b8bbae5724177ba16577be40d3aea] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp, totalSize=51.5 K 2024-11-10T03:16:19,878 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 67b942361abf4722b46bb67189b79d92, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731208576695 2024-11-10T03:16:19,880 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting c5f104db49ae4b59ba400cb62edc6433, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731208577092 2024-11-10T03:16:19,881 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting f4be510c6299423b9337909a8a491132, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731208577486 2024-11-10T03:16:19,883 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting d56b8bbae5724177ba16577be40d3aea, keycount=200, bloomtype=ROW, size=14.1 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731208579201 2024-11-10T03:16:19,916 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 77ed36df592326523261438e65e5b722 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-10T03:16:19,916 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77ed36df592326523261438e65e5b722, store=A 2024-11-10T03:16:19,916 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:16:19,916 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77ed36df592326523261438e65e5b722, store=B 2024-11-10T03:16:19,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:16:19,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on 77ed36df592326523261438e65e5b722 2024-11-10T03:16:19,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77ed36df592326523261438e65e5b722, store=C 2024-11-10T03:16:19,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:16:19,947 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/A/23440f3e46be452a9cb836b5053d0410 is 50, key is test_row_0/A:col10/1731208579559/Put/seqid=0 2024-11-10T03:16:19,953 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 77ed36df592326523261438e65e5b722#B#compaction#12 average throughput is 0.34 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:16:19,954 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/B/ecf1fba204b1447aa8318898c5576ac1 is 50, key is test_row_0/B:col10/1731208579239/Put/seqid=0 2024-11-10T03:16:19,956 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 77ed36df592326523261438e65e5b722#A#compaction#14 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:16:19,957 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/A/64d58ffdf77b4ba9b1d97305aff60dc9 is 50, key is test_row_0/A:col10/1731208579239/Put/seqid=0 2024-11-10T03:16:19,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741849_1025 (size=14541) 2024-11-10T03:16:19,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741850_1026 (size=12189) 2024-11-10T03:16:20,013 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/B/ecf1fba204b1447aa8318898c5576ac1 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/ecf1fba204b1447aa8318898c5576ac1 2024-11-10T03:16:20,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741851_1027 (size=12189) 2024-11-10T03:16:20,043 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/A/64d58ffdf77b4ba9b1d97305aff60dc9 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/64d58ffdf77b4ba9b1d97305aff60dc9 2024-11-10T03:16:20,052 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 77ed36df592326523261438e65e5b722/B of 77ed36df592326523261438e65e5b722 into ecf1fba204b1447aa8318898c5576ac1(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:16:20,052 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 77ed36df592326523261438e65e5b722: 2024-11-10T03:16:20,062 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731208579988, max timestamp=1731208580057Num uniques 859; 2024-11-10T03:16:20,062 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731208579988, max timestamp=1731208580057Num uniques 859; 2024-11-10T03:16:20,062 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-10T03:16:20,062 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-10T03:16:20,062 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731208579988, max timestamp=1731208580057Num uniques 859; 2024-11-10T03:16:20,063 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-10T03:16:20,065 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 77ed36df592326523261438e65e5b722/A of 77ed36df592326523261438e65e5b722 into 64d58ffdf77b4ba9b1d97305aff60dc9(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:16:20,065 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 77ed36df592326523261438e65e5b722: 2024-11-10T03:16:20,066 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., storeName=77ed36df592326523261438e65e5b722/A, priority=12, startTime=1731208579845; duration=0sec 2024-11-10T03:16:20,067 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T03:16:20,067 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 77ed36df592326523261438e65e5b722:A 2024-11-10T03:16:20,068 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T03:16:20,069 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., storeName=77ed36df592326523261438e65e5b722/B, priority=12, startTime=1731208579848; duration=0sec 2024-11-10T03:16:20,071 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48054 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T03:16:20,070 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:16:20,072 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 77ed36df592326523261438e65e5b722:B 2024-11-10T03:16:20,077 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1541): 77ed36df592326523261438e65e5b722/C is initiating minor compaction (all files) 2024-11-10T03:16:20,077 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 77ed36df592326523261438e65e5b722/C in TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:20,077 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/f79000458af44e5c9770b1f5f651e383, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/a70057a05b2547d8a434878573a74db6, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/c857d14211d94a209f81c919a922c5e5, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/9636f7c4fef04615b25eac6614ad11b2] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp, totalSize=46.9 K 2024-11-10T03:16:20,078 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting f79000458af44e5c9770b1f5f651e383, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731208576722 2024-11-10T03:16:20,080 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting a70057a05b2547d8a434878573a74db6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731208577092 2024-11-10T03:16:20,081 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting c857d14211d94a209f81c919a922c5e5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731208577486 2024-11-10T03:16:20,082 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9636f7c4fef04615b25eac6614ad11b2, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731208579201 2024-11-10T03:16:20,111 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 77ed36df592326523261438e65e5b722#C#compaction#15 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:16:20,112 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/C/6f1248b6174342508ee260e35805ff00 is 50, key is test_row_0/C:col10/1731208579239/Put/seqid=0 2024-11-10T03:16:20,141 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:20,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51370 deadline: 1731208640121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:20,143 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:20,143 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:20,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:20,143 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:20,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51378 deadline: 1731208640123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:20,144 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:20,144 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:20,144 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:20,148 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:20,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51402 deadline: 1731208640131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:20,149 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:20,149 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:20,149 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:20,149 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:20,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51380 deadline: 1731208640131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:20,150 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:20,150 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:20,150 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:20,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741852_1028 (size=12189) 2024-11-10T03:16:20,183 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/C/6f1248b6174342508ee260e35805ff00 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/6f1248b6174342508ee260e35805ff00 2024-11-10T03:16:20,197 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 77ed36df592326523261438e65e5b722/C of 77ed36df592326523261438e65e5b722 into 6f1248b6174342508ee260e35805ff00(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:16:20,197 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 77ed36df592326523261438e65e5b722: 2024-11-10T03:16:20,197 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., storeName=77ed36df592326523261438e65e5b722/C, priority=12, startTime=1731208579849; duration=0sec 2024-11-10T03:16:20,197 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:16:20,198 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 77ed36df592326523261438e65e5b722:C 2024-11-10T03:16:20,214 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:20,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51354 deadline: 1731208640214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:20,215 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:20,216 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:20,216 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:20,230 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-10T03:16:20,264 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:20,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51378 deadline: 1731208640253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:20,270 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:20,271 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:20,271 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:20,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:20,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51402 deadline: 1731208640254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:20,276 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:20,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51370 deadline: 1731208640254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:20,278 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:20,278 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:20,278 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:20,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:20,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51380 deadline: 1731208640254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:20,280 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:20,281 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:20,281 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:20,288 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:20,289 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:20,289 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:20,384 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/A/23440f3e46be452a9cb836b5053d0410 2024-11-10T03:16:20,411 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/B/8a6802dd4c034e5697d930ab594c0dfc is 50, key is test_row_0/B:col10/1731208579559/Put/seqid=0 2024-11-10T03:16:20,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741853_1029 (size=12151) 2024-11-10T03:16:20,464 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/B/8a6802dd4c034e5697d930ab594c0dfc 2024-11-10T03:16:20,485 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:20,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51378 deadline: 1731208640473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:20,486 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:20,486 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:20,486 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:20,487 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:20,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51402 deadline: 1731208640483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:20,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:20,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:20,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:20,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:20,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51380 deadline: 1731208640493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:20,503 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:20,503 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/C/dd40da2f209348c9b4fb55c6165fd802 is 50, key is test_row_0/C:col10/1731208579559/Put/seqid=0 2024-11-10T03:16:20,503 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:20,503 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:20,503 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:20,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51370 deadline: 1731208640494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:20,504 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:20,504 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:20,504 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:20,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741854_1030 (size=12151) 2024-11-10T03:16:20,548 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/C/dd40da2f209348c9b4fb55c6165fd802 2024-11-10T03:16:20,566 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/A/23440f3e46be452a9cb836b5053d0410 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/23440f3e46be452a9cb836b5053d0410 2024-11-10T03:16:20,579 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/23440f3e46be452a9cb836b5053d0410, entries=200, sequenceid=140, filesize=14.2 K 2024-11-10T03:16:20,594 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/B/8a6802dd4c034e5697d930ab594c0dfc as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/8a6802dd4c034e5697d930ab594c0dfc 2024-11-10T03:16:20,616 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/8a6802dd4c034e5697d930ab594c0dfc, entries=150, sequenceid=140, filesize=11.9 K 2024-11-10T03:16:20,618 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/C/dd40da2f209348c9b4fb55c6165fd802 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/dd40da2f209348c9b4fb55c6165fd802 2024-11-10T03:16:20,633 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/dd40da2f209348c9b4fb55c6165fd802, entries=150, sequenceid=140, filesize=11.9 K 2024-11-10T03:16:20,635 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 77ed36df592326523261438e65e5b722 in 718ms, sequenceid=140, compaction requested=false 2024-11-10T03:16:20,635 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 77ed36df592326523261438e65e5b722: 2024-11-10T03:16:20,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on 77ed36df592326523261438e65e5b722 2024-11-10T03:16:20,808 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 77ed36df592326523261438e65e5b722 3/3 column families, dataSize=147.60 KB heapSize=415.45 KB 2024-11-10T03:16:20,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77ed36df592326523261438e65e5b722, store=A 2024-11-10T03:16:20,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T03:16:20,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77ed36df592326523261438e65e5b722, store=B 2024-11-10T03:16:20,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T03:16:20,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77ed36df592326523261438e65e5b722, store=C 2024-11-10T03:16:20,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T03:16:20,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-10T03:16:20,824 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/A/78728cc51c274beca3469b062eb28e4e is 50, key is test_row_0/A:col10/1731208580123/Put/seqid=0 2024-11-10T03:16:20,825 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T03:16:20,826 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T03:16:20,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-10T03:16:20,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-10T03:16:20,830 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T03:16:20,831 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T03:16:20,831 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T03:16:20,848 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:20,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51378 deadline: 1731208640822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:20,850 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:20,850 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:20,850 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:20,859 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:20,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51370 deadline: 1731208640835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:20,861 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:20,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51380 deadline: 1731208640846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:20,863 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:20,863 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:20,863 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:20,863 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:20,863 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:20,863 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:20,864 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:20,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51402 deadline: 1731208640849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:20,865 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:20,865 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:20,865 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:20,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741855_1031 (size=16931) 2024-11-10T03:16:20,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-10T03:16:20,962 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:20,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51378 deadline: 1731208640954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:20,963 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:20,963 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:20,963 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:20,964 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-10T03:16:20,978 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:20,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51370 deadline: 1731208640973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:20,979 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:20,979 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:20,979 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:20,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51380 deadline: 1731208640973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:20,979 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:20,980 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:20,980 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:20,980 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:20,980 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:20,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51402 deadline: 1731208640973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:20,980 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:20,980 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:20,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:20,987 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-10T03:16:20,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:20,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. as already flushing 2024-11-10T03:16:20,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:20,988 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:20,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:20,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:21,144 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-10T03:16:21,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:21,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. as already flushing 2024-11-10T03:16:21,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:21,145 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:21,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:21,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-10T03:16:21,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:21,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:21,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51378 deadline: 1731208641173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:21,187 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:21,188 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:21,188 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:21,191 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:21,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51380 deadline: 1731208641184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:21,193 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:21,193 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:21,193 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:21,193 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:21,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51402 deadline: 1731208641184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:21,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:21,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51370 deadline: 1731208641184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:21,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:21,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:21,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:21,195 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:21,196 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:21,196 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:21,229 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:21,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51354 deadline: 1731208641223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:21,234 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 394d2222f724,38139,1731208572054 for 'test_row_1' in 77ed36df592326523261438e65e5b722 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2204 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T03:16:21,235 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:21,235 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:21,235 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:21,277 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=180 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/A/78728cc51c274beca3469b062eb28e4e 2024-11-10T03:16:21,300 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-10T03:16:21,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:21,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. as already flushing 2024-11-10T03:16:21,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:21,301 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/B/638046dd97a74b329485a9c6cfe40515 is 50, key is test_row_0/B:col10/1731208580123/Put/seqid=0 2024-11-10T03:16:21,301 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:21,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:21,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:21,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741856_1032 (size=12151) 2024-11-10T03:16:21,349 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=180 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/B/638046dd97a74b329485a9c6cfe40515 2024-11-10T03:16:21,384 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/C/c8a45b47e177497aafa40a0f7c8d48a3 is 50, key is test_row_0/C:col10/1731208580123/Put/seqid=0 2024-11-10T03:16:21,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741857_1033 (size=12151) 2024-11-10T03:16:21,418 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=180 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/C/c8a45b47e177497aafa40a0f7c8d48a3 2024-11-10T03:16:21,437 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/A/78728cc51c274beca3469b062eb28e4e as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/78728cc51c274beca3469b062eb28e4e 2024-11-10T03:16:21,450 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/78728cc51c274beca3469b062eb28e4e, entries=250, sequenceid=180, filesize=16.5 K 2024-11-10T03:16:21,452 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/B/638046dd97a74b329485a9c6cfe40515 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/638046dd97a74b329485a9c6cfe40515 2024-11-10T03:16:21,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-10T03:16:21,460 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-10T03:16:21,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:21,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. as already flushing 2024-11-10T03:16:21,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:21,460 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:21,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:21,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:21,464 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/638046dd97a74b329485a9c6cfe40515, entries=150, sequenceid=180, filesize=11.9 K 2024-11-10T03:16:21,468 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/C/c8a45b47e177497aafa40a0f7c8d48a3 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/c8a45b47e177497aafa40a0f7c8d48a3 2024-11-10T03:16:21,484 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/c8a45b47e177497aafa40a0f7c8d48a3, entries=150, sequenceid=180, filesize=11.9 K 2024-11-10T03:16:21,487 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~415.41 KB/425376, currentSize=33.54 KB/34350 for 77ed36df592326523261438e65e5b722 in 679ms, sequenceid=180, compaction requested=true 2024-11-10T03:16:21,488 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 77ed36df592326523261438e65e5b722: 2024-11-10T03:16:21,488 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 77ed36df592326523261438e65e5b722:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T03:16:21,488 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:16:21,488 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T03:16:21,488 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T03:16:21,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 77ed36df592326523261438e65e5b722:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T03:16:21,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:16:21,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 77ed36df592326523261438e65e5b722:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T03:16:21,490 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T03:16:21,490 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36491 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T03:16:21,490 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1541): 77ed36df592326523261438e65e5b722/B is initiating minor compaction (all files) 2024-11-10T03:16:21,490 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 77ed36df592326523261438e65e5b722/B in TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:21,491 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/ecf1fba204b1447aa8318898c5576ac1, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/8a6802dd4c034e5697d930ab594c0dfc, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/638046dd97a74b329485a9c6cfe40515] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp, totalSize=35.6 K 2024-11-10T03:16:21,491 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43661 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T03:16:21,492 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1541): 77ed36df592326523261438e65e5b722/A is initiating minor compaction (all files) 2024-11-10T03:16:21,492 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 77ed36df592326523261438e65e5b722/A in TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:21,492 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/64d58ffdf77b4ba9b1d97305aff60dc9, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/23440f3e46be452a9cb836b5053d0410, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/78728cc51c274beca3469b062eb28e4e] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp, totalSize=42.6 K 2024-11-10T03:16:21,492 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting ecf1fba204b1447aa8318898c5576ac1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731208579201 2024-11-10T03:16:21,498 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 64d58ffdf77b4ba9b1d97305aff60dc9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731208579201 2024-11-10T03:16:21,499 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a6802dd4c034e5697d930ab594c0dfc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1731208579551 2024-11-10T03:16:21,500 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 23440f3e46be452a9cb836b5053d0410, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1731208579545 2024-11-10T03:16:21,501 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 638046dd97a74b329485a9c6cfe40515, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1731208580095 2024-11-10T03:16:21,503 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 78728cc51c274beca3469b062eb28e4e, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1731208580095 2024-11-10T03:16:21,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on 77ed36df592326523261438e65e5b722 2024-11-10T03:16:21,507 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 77ed36df592326523261438e65e5b722 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-10T03:16:21,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77ed36df592326523261438e65e5b722, store=A 2024-11-10T03:16:21,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:16:21,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77ed36df592326523261438e65e5b722, store=B 2024-11-10T03:16:21,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:16:21,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77ed36df592326523261438e65e5b722, store=C 2024-11-10T03:16:21,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:16:21,525 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 77ed36df592326523261438e65e5b722#A#compaction#21 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:16:21,526 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/A/5ce8d689f149415e9a6ec3b82e7705aa is 50, key is test_row_0/A:col10/1731208580123/Put/seqid=0 2024-11-10T03:16:21,534 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/A/05da6396202d45bf88a07b86dd9a80fc is 50, key is test_row_0/A:col10/1731208581502/Put/seqid=0 2024-11-10T03:16:21,540 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 77ed36df592326523261438e65e5b722#B#compaction#23 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:16:21,541 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/B/9524576126ef48498ed311aac0243f91 is 50, key is test_row_0/B:col10/1731208580123/Put/seqid=0 2024-11-10T03:16:21,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741858_1034 (size=12391) 2024-11-10T03:16:21,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741859_1035 (size=16931) 2024-11-10T03:16:21,597 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/A/05da6396202d45bf88a07b86dd9a80fc 2024-11-10T03:16:21,613 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1731208581554, max timestamp=1731208581609Num uniques 859; 2024-11-10T03:16:21,613 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1731208581554, max timestamp=1731208581609Num uniques 859; 2024-11-10T03:16:21,613 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1731208581554, max timestamp=1731208581609Num uniques 859; 2024-11-10T03:16:21,613 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-11-10T03:16:21,613 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-11-10T03:16:21,613 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-11-10T03:16:21,617 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-10T03:16:21,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:21,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. as already flushing 2024-11-10T03:16:21,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:21,618 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:21,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:21,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:21,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741860_1036 (size=12391) 2024-11-10T03:16:21,625 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/B/9627444047fa4fe2865f20f167afcf0d is 50, key is test_row_0/B:col10/1731208581502/Put/seqid=0 2024-11-10T03:16:21,649 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/B/9524576126ef48498ed311aac0243f91 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/9524576126ef48498ed311aac0243f91 2024-11-10T03:16:21,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741861_1037 (size=12151) 2024-11-10T03:16:21,662 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/B/9627444047fa4fe2865f20f167afcf0d 2024-11-10T03:16:21,664 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-10T03:16:21,665 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-10T03:16:21,667 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-10T03:16:21,667 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-10T03:16:21,668 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-10T03:16:21,668 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-10T03:16:21,669 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 77ed36df592326523261438e65e5b722/B of 77ed36df592326523261438e65e5b722 into 9524576126ef48498ed311aac0243f91(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:16:21,669 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 77ed36df592326523261438e65e5b722: 2024-11-10T03:16:21,670 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., storeName=77ed36df592326523261438e65e5b722/B, priority=13, startTime=1731208581488; duration=0sec 2024-11-10T03:16:21,670 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T03:16:21,670 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 77ed36df592326523261438e65e5b722:B 2024-11-10T03:16:21,670 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T03:16:21,674 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-10T03:16:21,674 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-10T03:16:21,678 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36491 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T03:16:21,678 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1541): 77ed36df592326523261438e65e5b722/C is initiating minor compaction (all files) 2024-11-10T03:16:21,678 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 77ed36df592326523261438e65e5b722/C in TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:21,678 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/6f1248b6174342508ee260e35805ff00, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/dd40da2f209348c9b4fb55c6165fd802, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/c8a45b47e177497aafa40a0f7c8d48a3] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp, totalSize=35.6 K 2024-11-10T03:16:21,679 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f1248b6174342508ee260e35805ff00, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731208579201 2024-11-10T03:16:21,681 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting dd40da2f209348c9b4fb55c6165fd802, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1731208579551 2024-11-10T03:16:21,684 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting c8a45b47e177497aafa40a0f7c8d48a3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1731208580095 2024-11-10T03:16:21,693 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/C/dbd12687927e4e4f9955c5defbae5203 is 50, key is test_row_0/C:col10/1731208581502/Put/seqid=0 2024-11-10T03:16:21,712 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 77ed36df592326523261438e65e5b722#C#compaction#26 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:16:21,713 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/C/5cde76b584874ebbbc85d3b6e9e02330 is 50, key is test_row_0/C:col10/1731208580123/Put/seqid=0 2024-11-10T03:16:21,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:21,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51378 deadline: 1731208641709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:21,727 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:21,727 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:21,727 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:21,731 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:21,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51370 deadline: 1731208641712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:21,733 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:21,733 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:21,733 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:21,735 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:21,735 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:21,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51402 deadline: 1731208641717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:21,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51380 deadline: 1731208641712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:21,737 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:21,737 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:21,737 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:21,737 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:21,737 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:21,737 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:21,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741862_1038 (size=12151) 2024-11-10T03:16:21,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741863_1039 (size=12391) 2024-11-10T03:16:21,777 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-10T03:16:21,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:21,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. as already flushing 2024-11-10T03:16:21,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:21,778 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:21,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:21,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:21,793 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/C/5cde76b584874ebbbc85d3b6e9e02330 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/5cde76b584874ebbbc85d3b6e9e02330 2024-11-10T03:16:21,806 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 77ed36df592326523261438e65e5b722/C of 77ed36df592326523261438e65e5b722 into 5cde76b584874ebbbc85d3b6e9e02330(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:16:21,807 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 77ed36df592326523261438e65e5b722: 2024-11-10T03:16:21,807 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., storeName=77ed36df592326523261438e65e5b722/C, priority=13, startTime=1731208581489; duration=0sec 2024-11-10T03:16:21,807 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:16:21,807 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 77ed36df592326523261438e65e5b722:C 2024-11-10T03:16:21,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:21,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51378 deadline: 1731208641833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:21,838 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:21,838 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:21,839 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:21,857 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:21,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51370 deadline: 1731208641844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:21,858 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:21,858 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:21,858 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:21,858 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:21,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51402 deadline: 1731208641845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:21,859 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:21,859 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:21,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:21,860 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:21,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51380 deadline: 1731208641845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:21,861 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:21,861 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:21,861 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:21,935 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-10T03:16:21,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:21,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. as already flushing 2024-11-10T03:16:21,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:21,935 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:21,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:21,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:21,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-10T03:16:21,973 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/A/5ce8d689f149415e9a6ec3b82e7705aa as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/5ce8d689f149415e9a6ec3b82e7705aa 2024-11-10T03:16:21,990 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 77ed36df592326523261438e65e5b722/A of 77ed36df592326523261438e65e5b722 into 5ce8d689f149415e9a6ec3b82e7705aa(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:16:21,991 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 77ed36df592326523261438e65e5b722: 2024-11-10T03:16:21,991 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., storeName=77ed36df592326523261438e65e5b722/A, priority=13, startTime=1731208581488; duration=0sec 2024-11-10T03:16:21,991 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:16:21,991 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 77ed36df592326523261438e65e5b722:A 2024-11-10T03:16:22,049 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:22,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51378 deadline: 1731208642044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:22,052 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:22,052 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:22,052 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:22,064 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:22,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51370 deadline: 1731208642063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:22,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:22,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51402 deadline: 1731208642063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:22,066 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:22,066 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:22,066 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:22,067 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:22,067 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:22,067 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:22,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:22,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51380 deadline: 1731208642073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:22,080 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:22,080 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:22,080 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:22,092 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-10T03:16:22,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:22,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. as already flushing 2024-11-10T03:16:22,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:22,093 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:22,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:22,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:22,156 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/C/dbd12687927e4e4f9955c5defbae5203 2024-11-10T03:16:22,170 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/A/05da6396202d45bf88a07b86dd9a80fc as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/05da6396202d45bf88a07b86dd9a80fc 2024-11-10T03:16:22,180 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/05da6396202d45bf88a07b86dd9a80fc, entries=250, sequenceid=190, filesize=16.5 K 2024-11-10T03:16:22,182 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/B/9627444047fa4fe2865f20f167afcf0d as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/9627444047fa4fe2865f20f167afcf0d 2024-11-10T03:16:22,192 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/9627444047fa4fe2865f20f167afcf0d, entries=150, sequenceid=190, filesize=11.9 K 2024-11-10T03:16:22,195 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/C/dbd12687927e4e4f9955c5defbae5203 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/dbd12687927e4e4f9955c5defbae5203 2024-11-10T03:16:22,226 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/dbd12687927e4e4f9955c5defbae5203, entries=150, sequenceid=190, filesize=11.9 K 2024-11-10T03:16:22,229 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=147.60 KB/151140 for 77ed36df592326523261438e65e5b722 in 723ms, sequenceid=190, compaction requested=false 2024-11-10T03:16:22,229 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 77ed36df592326523261438e65e5b722: 2024-11-10T03:16:22,248 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-10T03:16:22,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:22,249 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 77ed36df592326523261438e65e5b722 3/3 column families, dataSize=147.60 KB heapSize=401.39 KB 2024-11-10T03:16:22,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77ed36df592326523261438e65e5b722, store=A 2024-11-10T03:16:22,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T03:16:22,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77ed36df592326523261438e65e5b722, store=B 2024-11-10T03:16:22,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T03:16:22,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77ed36df592326523261438e65e5b722, store=C 2024-11-10T03:16:22,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T03:16:22,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/A/655e7a1e8dd24177a4bf8dc70955149c is 50, key is test_row_0/A:col10/1731208581708/Put/seqid=0 2024-11-10T03:16:22,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741864_1040 (size=12151) 2024-11-10T03:16:22,369 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. as already flushing 2024-11-10T03:16:22,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on 77ed36df592326523261438e65e5b722 2024-11-10T03:16:22,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:22,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51380 deadline: 1731208642383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:22,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:22,391 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:22,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51370 deadline: 1731208642386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:22,391 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:22,392 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:22,392 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:22,392 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:22,392 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:22,402 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:22,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51402 deadline: 1731208642392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:22,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:22,403 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:22,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:22,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:22,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51378 deadline: 1731208642392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:22,404 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:22,404 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:22,405 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:22,507 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:22,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51370 deadline: 1731208642503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:22,508 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:22,508 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:22,508 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:22,514 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:22,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51402 deadline: 1731208642513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:22,515 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:22,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51378 deadline: 1731208642513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:22,516 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:22,516 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:22,516 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:22,517 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:22,517 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:22,517 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:22,676 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/A/655e7a1e8dd24177a4bf8dc70955149c 2024-11-10T03:16:22,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/B/c70b7d9bc2614bb985329243a5c9ae0f is 50, key is test_row_0/B:col10/1731208581708/Put/seqid=0 2024-11-10T03:16:22,716 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:22,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51370 deadline: 1731208642713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:22,717 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:22,717 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:22,717 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:22,726 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:22,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51378 deadline: 1731208642723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:22,727 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:22,727 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:22,727 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:22,728 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:22,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51402 deadline: 1731208642724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:22,728 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:22,728 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:22,728 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:22,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741865_1041 (size=12151) 2024-11-10T03:16:22,909 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:22,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51380 deadline: 1731208642903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:22,913 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:22,914 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:22,914 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:22,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-10T03:16:23,024 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:23,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51370 deadline: 1731208643023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:23,027 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:23,027 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:23,027 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:23,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:23,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51378 deadline: 1731208643032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:23,042 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:23,042 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:23,042 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:23,042 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:23,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51402 deadline: 1731208643033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:23,047 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:23,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:23,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:23,144 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/B/c70b7d9bc2614bb985329243a5c9ae0f 2024-11-10T03:16:23,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/C/a9daa8f6c68044c4865965f796c44c9e is 50, key is test_row_0/C:col10/1731208581708/Put/seqid=0 2024-11-10T03:16:23,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741866_1042 (size=12151) 2024-11-10T03:16:23,203 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/C/a9daa8f6c68044c4865965f796c44c9e 2024-11-10T03:16:23,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/A/655e7a1e8dd24177a4bf8dc70955149c as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/655e7a1e8dd24177a4bf8dc70955149c 2024-11-10T03:16:23,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:23,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51354 deadline: 1731208643243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:23,249 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 394d2222f724,38139,1731208572054 for 'test_row_1' in 77ed36df592326523261438e65e5b722 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4222 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T03:16:23,250 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:23,250 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:23,250 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:23,252 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/655e7a1e8dd24177a4bf8dc70955149c, entries=150, sequenceid=227, filesize=11.9 K 2024-11-10T03:16:23,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/B/c70b7d9bc2614bb985329243a5c9ae0f as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/c70b7d9bc2614bb985329243a5c9ae0f 2024-11-10T03:16:23,266 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/c70b7d9bc2614bb985329243a5c9ae0f, entries=150, sequenceid=227, filesize=11.9 K 2024-11-10T03:16:23,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/C/a9daa8f6c68044c4865965f796c44c9e as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/a9daa8f6c68044c4865965f796c44c9e 2024-11-10T03:16:23,281 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/a9daa8f6c68044c4865965f796c44c9e, entries=150, sequenceid=227, filesize=11.9 K 2024-11-10T03:16:23,283 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~401.34 KB/410976, currentSize=40.25 KB/41220 for 77ed36df592326523261438e65e5b722 in 1035ms, sequenceid=227, compaction requested=true 2024-11-10T03:16:23,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 77ed36df592326523261438e65e5b722: 2024-11-10T03:16:23,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:23,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-10T03:16:23,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37299 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-10T03:16:23,290 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-10T03:16:23,290 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4550 sec 2024-11-10T03:16:23,295 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 2.4650 sec 2024-11-10T03:16:23,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on 77ed36df592326523261438e65e5b722 2024-11-10T03:16:23,545 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 77ed36df592326523261438e65e5b722 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-10T03:16:23,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77ed36df592326523261438e65e5b722, store=A 2024-11-10T03:16:23,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:16:23,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77ed36df592326523261438e65e5b722, store=B 2024-11-10T03:16:23,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:16:23,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77ed36df592326523261438e65e5b722, store=C 2024-11-10T03:16:23,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:16:23,555 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/A/a1ddef07d63b4a548613f14785b28374 is 50, key is test_row_0/A:col10/1731208582387/Put/seqid=0 2024-11-10T03:16:23,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741867_1043 (size=12147) 2024-11-10T03:16:23,630 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731208583612, max timestamp=1731208583622Num uniques 859; 2024-11-10T03:16:23,630 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T03:16:23,632 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731208583612, max timestamp=1731208583622Num uniques 859; 2024-11-10T03:16:23,632 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731208583612, max timestamp=1731208583622Num uniques 859; 2024-11-10T03:16:23,633 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T03:16:23,633 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T03:16:23,694 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:23,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51370 deadline: 1731208643688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:23,695 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:23,695 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:23,695 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:23,696 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:23,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51402 deadline: 1731208643688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:23,696 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:23,696 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:23,696 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:23,696 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:23,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51378 deadline: 1731208643688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:23,697 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:23,697 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:23,697 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:23,810 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:23,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51370 deadline: 1731208643803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:23,811 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:23,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51402 deadline: 1731208643804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:23,812 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:23,812 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:23,812 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:23,817 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:23,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51378 deadline: 1731208643810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:23,818 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:23,818 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:23,818 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:23,819 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:23,819 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:23,819 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:23,926 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:23,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51380 deadline: 1731208643923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:23,928 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 394d2222f724,38139,1731208572054 for 'test_row_1' in 77ed36df592326523261438e65e5b722 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2217 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T03:16:23,929 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:23,929 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:23,929 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:24,009 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/A/a1ddef07d63b4a548613f14785b28374 2024-11-10T03:16:24,028 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/B/852f97728a4e49d18f38f2da39fd4ba2 is 50, key is test_row_0/B:col10/1731208582387/Put/seqid=0 2024-11-10T03:16:24,029 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:24,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51402 deadline: 1731208644023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:24,030 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:24,030 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:24,030 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:24,030 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:24,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51370 deadline: 1731208644024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:24,031 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:24,031 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:24,031 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:24,031 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:24,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51378 deadline: 1731208644024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:24,032 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:24,032 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:24,032 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:24,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741868_1044 (size=9757) 2024-11-10T03:16:24,062 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/B/852f97728a4e49d18f38f2da39fd4ba2 2024-11-10T03:16:24,088 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/C/3c64207948624446bc587a1dc3b053a6 is 50, key is test_row_0/C:col10/1731208582387/Put/seqid=0 2024-11-10T03:16:24,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741869_1045 (size=9757) 2024-11-10T03:16:24,335 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:24,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51402 deadline: 1731208644333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:24,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:24,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:24,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:24,353 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:24,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51370 deadline: 1731208644344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:24,354 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:24,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51378 deadline: 1731208644344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:24,354 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:24,354 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:24,354 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:24,354 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:24,354 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:24,355 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:24,511 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/C/3c64207948624446bc587a1dc3b053a6 2024-11-10T03:16:24,527 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/A/a1ddef07d63b4a548613f14785b28374 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/a1ddef07d63b4a548613f14785b28374 2024-11-10T03:16:24,538 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/a1ddef07d63b4a548613f14785b28374, entries=150, sequenceid=237, filesize=11.9 K 2024-11-10T03:16:24,539 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/B/852f97728a4e49d18f38f2da39fd4ba2 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/852f97728a4e49d18f38f2da39fd4ba2 2024-11-10T03:16:24,552 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/852f97728a4e49d18f38f2da39fd4ba2, entries=100, sequenceid=237, filesize=9.5 K 2024-11-10T03:16:24,555 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/C/3c64207948624446bc587a1dc3b053a6 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/3c64207948624446bc587a1dc3b053a6 2024-11-10T03:16:24,568 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/3c64207948624446bc587a1dc3b053a6, entries=100, sequenceid=237, filesize=9.5 K 2024-11-10T03:16:24,570 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 77ed36df592326523261438e65e5b722 in 1025ms, sequenceid=237, compaction requested=true 2024-11-10T03:16:24,570 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 77ed36df592326523261438e65e5b722: 2024-11-10T03:16:24,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 77ed36df592326523261438e65e5b722:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T03:16:24,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:16:24,570 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T03:16:24,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 77ed36df592326523261438e65e5b722:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T03:16:24,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T03:16:24,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 77ed36df592326523261438e65e5b722:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T03:16:24,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T03:16:24,571 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T03:16:24,573 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53620 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T03:16:24,573 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1541): 77ed36df592326523261438e65e5b722/A is initiating minor compaction (all files) 2024-11-10T03:16:24,573 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 77ed36df592326523261438e65e5b722/A in TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:24,573 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/5ce8d689f149415e9a6ec3b82e7705aa, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/05da6396202d45bf88a07b86dd9a80fc, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/655e7a1e8dd24177a4bf8dc70955149c, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/a1ddef07d63b4a548613f14785b28374] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp, totalSize=52.4 K 2024-11-10T03:16:24,574 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ce8d689f149415e9a6ec3b82e7705aa, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1731208580095 2024-11-10T03:16:24,575 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46450 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T03:16:24,575 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1541): 77ed36df592326523261438e65e5b722/B is initiating minor compaction (all files) 2024-11-10T03:16:24,575 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 77ed36df592326523261438e65e5b722/B in TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:24,575 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/9524576126ef48498ed311aac0243f91, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/9627444047fa4fe2865f20f167afcf0d, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/c70b7d9bc2614bb985329243a5c9ae0f, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/852f97728a4e49d18f38f2da39fd4ba2] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp, totalSize=45.4 K 2024-11-10T03:16:24,576 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 05da6396202d45bf88a07b86dd9a80fc, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731208580816 2024-11-10T03:16:24,576 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 9524576126ef48498ed311aac0243f91, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1731208580095 2024-11-10T03:16:24,577 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 9627444047fa4fe2865f20f167afcf0d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731208580816 2024-11-10T03:16:24,577 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 655e7a1e8dd24177a4bf8dc70955149c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1731208581707 2024-11-10T03:16:24,577 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting c70b7d9bc2614bb985329243a5c9ae0f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1731208581707 2024-11-10T03:16:24,578 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting a1ddef07d63b4a548613f14785b28374, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731208582381 2024-11-10T03:16:24,579 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 852f97728a4e49d18f38f2da39fd4ba2, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731208582387 2024-11-10T03:16:24,605 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 77ed36df592326523261438e65e5b722#B#compaction#33 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:16:24,605 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/B/acfca18bf378472cadd8637388d72a74 is 50, key is test_row_0/B:col10/1731208582387/Put/seqid=0 2024-11-10T03:16:24,609 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 77ed36df592326523261438e65e5b722#A#compaction#34 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:16:24,612 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/A/60e12467571048228a11f35b56626d52 is 50, key is test_row_0/A:col10/1731208582387/Put/seqid=0 2024-11-10T03:16:24,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741870_1046 (size=12527) 2024-11-10T03:16:24,659 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/B/acfca18bf378472cadd8637388d72a74 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/acfca18bf378472cadd8637388d72a74 2024-11-10T03:16:24,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741871_1047 (size=12527) 2024-11-10T03:16:24,672 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 77ed36df592326523261438e65e5b722/B of 77ed36df592326523261438e65e5b722 into acfca18bf378472cadd8637388d72a74(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:16:24,672 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 77ed36df592326523261438e65e5b722: 2024-11-10T03:16:24,672 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., storeName=77ed36df592326523261438e65e5b722/B, priority=12, startTime=1731208584571; duration=0sec 2024-11-10T03:16:24,672 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T03:16:24,672 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 77ed36df592326523261438e65e5b722:B 2024-11-10T03:16:24,672 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T03:16:24,675 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46450 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T03:16:24,675 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1541): 77ed36df592326523261438e65e5b722/C is initiating minor compaction (all files) 2024-11-10T03:16:24,676 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 77ed36df592326523261438e65e5b722/C in TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:24,676 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/5cde76b584874ebbbc85d3b6e9e02330, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/dbd12687927e4e4f9955c5defbae5203, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/a9daa8f6c68044c4865965f796c44c9e, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/3c64207948624446bc587a1dc3b053a6] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp, totalSize=45.4 K 2024-11-10T03:16:24,684 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/A/60e12467571048228a11f35b56626d52 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/60e12467571048228a11f35b56626d52 2024-11-10T03:16:24,677 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 5cde76b584874ebbbc85d3b6e9e02330, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1731208580095 2024-11-10T03:16:24,691 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting dbd12687927e4e4f9955c5defbae5203, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731208580816 2024-11-10T03:16:24,694 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting a9daa8f6c68044c4865965f796c44c9e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1731208581707 2024-11-10T03:16:24,696 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c64207948624446bc587a1dc3b053a6, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731208582387 2024-11-10T03:16:24,703 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 77ed36df592326523261438e65e5b722/A of 77ed36df592326523261438e65e5b722 into 60e12467571048228a11f35b56626d52(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:16:24,703 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 77ed36df592326523261438e65e5b722: 2024-11-10T03:16:24,703 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., storeName=77ed36df592326523261438e65e5b722/A, priority=12, startTime=1731208584570; duration=0sec 2024-11-10T03:16:24,703 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:16:24,703 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 77ed36df592326523261438e65e5b722:A 2024-11-10T03:16:24,725 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 77ed36df592326523261438e65e5b722#C#compaction#35 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:16:24,726 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/C/18b79e975d43438d9ed669fbf4c38c3b is 50, key is test_row_0/C:col10/1731208582387/Put/seqid=0 2024-11-10T03:16:24,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741872_1048 (size=12527) 2024-11-10T03:16:24,764 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/C/18b79e975d43438d9ed669fbf4c38c3b as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/18b79e975d43438d9ed669fbf4c38c3b 2024-11-10T03:16:24,781 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 77ed36df592326523261438e65e5b722/C of 77ed36df592326523261438e65e5b722 into 18b79e975d43438d9ed669fbf4c38c3b(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:16:24,782 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 77ed36df592326523261438e65e5b722: 2024-11-10T03:16:24,782 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., storeName=77ed36df592326523261438e65e5b722/C, priority=12, startTime=1731208584571; duration=0sec 2024-11-10T03:16:24,782 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:16:24,782 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 77ed36df592326523261438e65e5b722:C 2024-11-10T03:16:24,854 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731208583662, max timestamp=1731208584847Num uniques 1018; 2024-11-10T03:16:24,854 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-10T03:16:24,854 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731208583662, max timestamp=1731208584847Num uniques 1018; 2024-11-10T03:16:24,855 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-10T03:16:24,858 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731208583662, max timestamp=1731208584847Num uniques 1018; 2024-11-10T03:16:24,859 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-10T03:16:24,879 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 77ed36df592326523261438e65e5b722 3/3 column families, dataSize=53.67 KB heapSize=145.08 KB 2024-11-10T03:16:24,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77ed36df592326523261438e65e5b722, store=A 2024-11-10T03:16:24,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T03:16:24,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77ed36df592326523261438e65e5b722, store=B 2024-11-10T03:16:24,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T03:16:24,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77ed36df592326523261438e65e5b722, store=C 2024-11-10T03:16:24,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T03:16:24,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on 77ed36df592326523261438e65e5b722 2024-11-10T03:16:24,901 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/A/dd96679e1c87434796be5697903956e5 is 50, key is test_row_0/A:col10/1731208583685/Put/seqid=0 2024-11-10T03:16:24,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741873_1049 (size=14741) 2024-11-10T03:16:24,985 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731208584935, max timestamp=1731208584967Num uniques 859; 2024-11-10T03:16:24,985 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-10T03:16:24,986 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731208584935, max timestamp=1731208584983Num uniques 859; 2024-11-10T03:16:24,986 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-10T03:16:24,986 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731208584935, max timestamp=1731208584967Num uniques 859; 2024-11-10T03:16:24,987 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-10T03:16:24,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-10T03:16:24,993 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T03:16:24,994 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37299 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T03:16:25,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37299 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-10T03:16:25,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-10T03:16:25,006 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T03:16:25,009 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T03:16:25,009 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T03:16:25,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:25,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51370 deadline: 1731208645104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:25,110 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:25,111 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:25,111 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:25,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-10T03:16:25,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:25,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51378 deadline: 1731208645107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:25,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:25,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51402 deadline: 1731208645108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:25,128 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:25,128 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:25,128 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:25,128 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:25,129 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:25,129 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:25,165 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-10T03:16:25,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:25,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. as already flushing 2024-11-10T03:16:25,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:25,166 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:25,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:25,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:25,230 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:25,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51370 deadline: 1731208645213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:25,231 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:25,231 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:25,231 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:25,239 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:25,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51402 deadline: 1731208645234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:25,240 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:25,240 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:25,239 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:25,240 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:25,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51378 deadline: 1731208645234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:25,240 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:25,241 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:25,241 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:25,322 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-10T03:16:25,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:25,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. as already flushing 2024-11-10T03:16:25,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:25,323 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:25,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-10T03:16:25,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:25,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:25,354 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/A/dd96679e1c87434796be5697903956e5 2024-11-10T03:16:25,384 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/B/239578cf7394453b807f66cc0d7133cb is 50, key is test_row_0/B:col10/1731208583685/Put/seqid=0 2024-11-10T03:16:25,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741874_1050 (size=12301) 2024-11-10T03:16:25,446 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/B/239578cf7394453b807f66cc0d7133cb 2024-11-10T03:16:25,450 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:25,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51370 deadline: 1731208645443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:25,451 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:25,451 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:25,451 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:25,451 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:25,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51402 deadline: 1731208645443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:25,452 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:25,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:25,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:25,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51378 deadline: 1731208645444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:25,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:25,453 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:25,453 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:25,453 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:25,463 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/C/3f00d2151e6c417c9815126b19d2de00 is 50, key is test_row_0/C:col10/1731208583685/Put/seqid=0 2024-11-10T03:16:25,478 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-10T03:16:25,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:25,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. as already flushing 2024-11-10T03:16:25,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:25,478 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:25,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:25,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:25,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741875_1051 (size=12301) 2024-11-10T03:16:25,495 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/C/3f00d2151e6c417c9815126b19d2de00 2024-11-10T03:16:25,507 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/A/dd96679e1c87434796be5697903956e5 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/dd96679e1c87434796be5697903956e5 2024-11-10T03:16:25,520 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/dd96679e1c87434796be5697903956e5, entries=200, sequenceid=282, filesize=14.4 K 2024-11-10T03:16:25,523 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/B/239578cf7394453b807f66cc0d7133cb as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/239578cf7394453b807f66cc0d7133cb 2024-11-10T03:16:25,535 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/239578cf7394453b807f66cc0d7133cb, entries=150, sequenceid=282, filesize=12.0 K 2024-11-10T03:16:25,539 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/C/3f00d2151e6c417c9815126b19d2de00 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/3f00d2151e6c417c9815126b19d2de00 2024-11-10T03:16:25,550 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/3f00d2151e6c417c9815126b19d2de00, entries=150, sequenceid=282, filesize=12.0 K 2024-11-10T03:16:25,551 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~145.03 KB/148512, currentSize=140.89 KB/144270 for 77ed36df592326523261438e65e5b722 in 673ms, sequenceid=282, compaction requested=false 2024-11-10T03:16:25,551 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 77ed36df592326523261438e65e5b722: 2024-11-10T03:16:25,632 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-10T03:16:25,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:25,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-10T03:16:25,633 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 77ed36df592326523261438e65e5b722 3/3 column families, dataSize=140.89 KB heapSize=395.53 KB 2024-11-10T03:16:25,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77ed36df592326523261438e65e5b722, store=A 2024-11-10T03:16:25,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T03:16:25,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77ed36df592326523261438e65e5b722, store=B 2024-11-10T03:16:25,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T03:16:25,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77ed36df592326523261438e65e5b722, store=C 2024-11-10T03:16:25,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T03:16:25,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/A/e9f5ad945db14b4c89effed4375ba61b is 50, key is test_row_0/A:col10/1731208585105/Put/seqid=0 2024-11-10T03:16:25,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741876_1052 (size=12301) 2024-11-10T03:16:25,652 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/A/e9f5ad945db14b4c89effed4375ba61b 2024-11-10T03:16:25,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/B/d12a9c8d0b4042ec9d3dc069b11b56a8 is 50, key is test_row_0/B:col10/1731208585105/Put/seqid=0 2024-11-10T03:16:25,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741877_1053 (size=12301) 2024-11-10T03:16:25,683 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/B/d12a9c8d0b4042ec9d3dc069b11b56a8 2024-11-10T03:16:25,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/C/64c4f6aee6f54bb9867f3eecfa40cdcb is 50, key is test_row_0/C:col10/1731208585105/Put/seqid=0 2024-11-10T03:16:25,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741878_1054 (size=12301) 2024-11-10T03:16:25,709 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/C/64c4f6aee6f54bb9867f3eecfa40cdcb 2024-11-10T03:16:25,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/A/e9f5ad945db14b4c89effed4375ba61b as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/e9f5ad945db14b4c89effed4375ba61b 2024-11-10T03:16:25,726 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/e9f5ad945db14b4c89effed4375ba61b, entries=150, sequenceid=318, filesize=12.0 K 2024-11-10T03:16:25,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/B/d12a9c8d0b4042ec9d3dc069b11b56a8 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/d12a9c8d0b4042ec9d3dc069b11b56a8 2024-11-10T03:16:25,734 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/d12a9c8d0b4042ec9d3dc069b11b56a8, entries=150, sequenceid=318, filesize=12.0 K 2024-11-10T03:16:25,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/C/64c4f6aee6f54bb9867f3eecfa40cdcb as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/64c4f6aee6f54bb9867f3eecfa40cdcb 2024-11-10T03:16:25,750 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/64c4f6aee6f54bb9867f3eecfa40cdcb, entries=150, sequenceid=318, filesize=12.0 K 2024-11-10T03:16:25,751 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~395.48 KB/404976, currentSize=0 B/0 for 77ed36df592326523261438e65e5b722 in 118ms, sequenceid=318, compaction requested=true 2024-11-10T03:16:25,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 77ed36df592326523261438e65e5b722: 2024-11-10T03:16:25,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:25,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-10T03:16:25,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37299 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-10T03:16:25,756 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-10T03:16:25,756 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 744 msec 2024-11-10T03:16:25,760 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 764 msec 2024-11-10T03:16:25,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on 77ed36df592326523261438e65e5b722 2024-11-10T03:16:25,777 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 77ed36df592326523261438e65e5b722 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-10T03:16:25,777 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77ed36df592326523261438e65e5b722, store=A 2024-11-10T03:16:25,777 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:16:25,777 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77ed36df592326523261438e65e5b722, store=B 2024-11-10T03:16:25,777 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:16:25,777 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77ed36df592326523261438e65e5b722, store=C 2024-11-10T03:16:25,778 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:16:25,802 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/A/62f0a998e40549b8911742c7d484727b is 50, key is test_row_0/A:col10/1731208585774/Put/seqid=0 2024-11-10T03:16:25,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741879_1055 (size=14741) 2024-11-10T03:16:25,813 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/A/62f0a998e40549b8911742c7d484727b 2024-11-10T03:16:25,830 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/B/e4c7aceabb9d4b6d9ec6ece931eba970 is 50, key is test_row_0/B:col10/1731208585774/Put/seqid=0 2024-11-10T03:16:25,838 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731208585797, max timestamp=1731208585834Num uniques 859; 2024-11-10T03:16:25,839 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731208585797, max timestamp=1731208585834Num uniques 859; 2024-11-10T03:16:25,839 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-10T03:16:25,839 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-10T03:16:25,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741880_1056 (size=12301) 2024-11-10T03:16:25,841 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731208585817, max timestamp=1731208585834Num uniques 859; 2024-11-10T03:16:25,841 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T03:16:25,903 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:25,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51370 deadline: 1731208645897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:25,904 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:25,904 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:25,904 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:25,913 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:25,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51402 deadline: 1731208645901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:25,915 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:25,916 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:25,915 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:25,916 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:25,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51378 deadline: 1731208645903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:25,916 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:25,916 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:25,916 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:25,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:25,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51380 deadline: 1731208645953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:25,961 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 394d2222f724,38139,1731208572054 for 'test_row_1' in 77ed36df592326523261438e65e5b722 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4250 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T03:16:25,962 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:25,962 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:25,962 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:26,015 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:26,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51370 deadline: 1731208646015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:26,020 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:26,020 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:26,020 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:26,028 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:26,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51378 deadline: 1731208646023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:26,029 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:26,029 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:26,029 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:26,029 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:26,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51402 deadline: 1731208646024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:26,030 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:26,030 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:26,030 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:26,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-10T03:16:26,156 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T03:16:26,157 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37299 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T03:16:26,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37299 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-10T03:16:26,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-10T03:16:26,164 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T03:16:26,166 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T03:16:26,166 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T03:16:26,223 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:26,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51370 deadline: 1731208646223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:26,224 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:26,224 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:26,224 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:26,241 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/B/e4c7aceabb9d4b6d9ec6ece931eba970 2024-11-10T03:16:26,243 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:26,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51378 deadline: 1731208646233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:26,244 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:26,244 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:26,244 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:26,244 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:26,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51402 deadline: 1731208646234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:26,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:26,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:26,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:26,267 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/C/c05ee47056fc4a1aab3294f34bf6271e is 50, key is test_row_0/C:col10/1731208585774/Put/seqid=0 2024-11-10T03:16:26,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-10T03:16:26,325 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-10T03:16:26,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:26,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. as already flushing 2024-11-10T03:16:26,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:26,326 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:26,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:26,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:26,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741881_1057 (size=12301) 2024-11-10T03:16:26,481 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-10T03:16:26,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:26,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. as already flushing 2024-11-10T03:16:26,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:26,482 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:26,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:26,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:26,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-10T03:16:26,535 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:26,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51370 deadline: 1731208646533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:26,536 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:26,536 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:26,536 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:26,558 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:26,559 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:26,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51378 deadline: 1731208646554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:26,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51402 deadline: 1731208646554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:26,560 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:26,560 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:26,560 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:26,560 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:26,560 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:26,560 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:26,641 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-10T03:16:26,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:26,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. as already flushing 2024-11-10T03:16:26,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:26,642 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:26,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:26,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:26,731 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/C/c05ee47056fc4a1aab3294f34bf6271e 2024-11-10T03:16:26,752 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/A/62f0a998e40549b8911742c7d484727b as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/62f0a998e40549b8911742c7d484727b 2024-11-10T03:16:26,770 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/62f0a998e40549b8911742c7d484727b, entries=200, sequenceid=328, filesize=14.4 K 2024-11-10T03:16:26,778 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/B/e4c7aceabb9d4b6d9ec6ece931eba970 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/e4c7aceabb9d4b6d9ec6ece931eba970 2024-11-10T03:16:26,789 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/e4c7aceabb9d4b6d9ec6ece931eba970, entries=150, sequenceid=328, filesize=12.0 K 2024-11-10T03:16:26,792 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/C/c05ee47056fc4a1aab3294f34bf6271e as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/c05ee47056fc4a1aab3294f34bf6271e 2024-11-10T03:16:26,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-10T03:16:26,796 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-10T03:16:26,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:26,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. as already flushing 2024-11-10T03:16:26,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:26,797 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:26,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:26,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:26,820 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/c05ee47056fc4a1aab3294f34bf6271e, entries=150, sequenceid=328, filesize=12.0 K 2024-11-10T03:16:26,821 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=136.42 KB/139690 for 77ed36df592326523261438e65e5b722 in 1044ms, sequenceid=328, compaction requested=true 2024-11-10T03:16:26,821 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 77ed36df592326523261438e65e5b722: 2024-11-10T03:16:26,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 77ed36df592326523261438e65e5b722:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T03:16:26,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:16:26,822 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T03:16:26,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 77ed36df592326523261438e65e5b722:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T03:16:26,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T03:16:26,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 77ed36df592326523261438e65e5b722:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T03:16:26,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-10T03:16:26,822 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T03:16:26,823 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54310 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T03:16:26,824 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1541): 77ed36df592326523261438e65e5b722/A is initiating minor compaction (all files) 2024-11-10T03:16:26,824 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49430 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T03:16:26,824 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 77ed36df592326523261438e65e5b722/A in TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:26,824 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1541): 77ed36df592326523261438e65e5b722/B is initiating minor compaction (all files) 2024-11-10T03:16:26,824 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 77ed36df592326523261438e65e5b722/B in TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:26,824 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/60e12467571048228a11f35b56626d52, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/dd96679e1c87434796be5697903956e5, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/e9f5ad945db14b4c89effed4375ba61b, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/62f0a998e40549b8911742c7d484727b] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp, totalSize=53.0 K 2024-11-10T03:16:26,824 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/acfca18bf378472cadd8637388d72a74, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/239578cf7394453b807f66cc0d7133cb, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/d12a9c8d0b4042ec9d3dc069b11b56a8, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/e4c7aceabb9d4b6d9ec6ece931eba970] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp, totalSize=48.3 K 2024-11-10T03:16:26,825 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting acfca18bf378472cadd8637388d72a74, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731208581709 2024-11-10T03:16:26,825 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 60e12467571048228a11f35b56626d52, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731208581709 2024-11-10T03:16:26,828 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 239578cf7394453b807f66cc0d7133cb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731208583685 2024-11-10T03:16:26,829 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting dd96679e1c87434796be5697903956e5, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731208583685 2024-11-10T03:16:26,830 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting d12a9c8d0b4042ec9d3dc069b11b56a8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1731208585076 2024-11-10T03:16:26,830 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting e9f5ad945db14b4c89effed4375ba61b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1731208585076 2024-11-10T03:16:26,830 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting e4c7aceabb9d4b6d9ec6ece931eba970, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731208585768 2024-11-10T03:16:26,830 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 62f0a998e40549b8911742c7d484727b, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731208585768 2024-11-10T03:16:26,859 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 77ed36df592326523261438e65e5b722#A#compaction#45 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:16:26,860 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/A/905a50fe0e4945c580f1203d43d41c3f is 50, key is test_row_0/A:col10/1731208585774/Put/seqid=0 2024-11-10T03:16:26,879 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 77ed36df592326523261438e65e5b722#B#compaction#46 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:16:26,880 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/B/bddf2f14f2d24a3ea6fa9994e7e6199a is 50, key is test_row_0/B:col10/1731208585774/Put/seqid=0 2024-11-10T03:16:26,972 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-10T03:16:26,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:26,972 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 77ed36df592326523261438e65e5b722 3/3 column families, dataSize=136.42 KB heapSize=385.38 KB 2024-11-10T03:16:26,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77ed36df592326523261438e65e5b722, store=A 2024-11-10T03:16:26,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T03:16:26,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77ed36df592326523261438e65e5b722, store=B 2024-11-10T03:16:26,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T03:16:26,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77ed36df592326523261438e65e5b722, store=C 2024-11-10T03:16:26,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T03:16:26,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741882_1058 (size=12813) 2024-11-10T03:16:26,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741883_1059 (size=12813) 2024-11-10T03:16:26,996 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/A/905a50fe0e4945c580f1203d43d41c3f as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/905a50fe0e4945c580f1203d43d41c3f 2024-11-10T03:16:26,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/A/523e5f6c57814d5490dbe2d0e58cf575 is 50, key is test_row_0/A:col10/1731208585901/Put/seqid=0 2024-11-10T03:16:27,006 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/B/bddf2f14f2d24a3ea6fa9994e7e6199a as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/bddf2f14f2d24a3ea6fa9994e7e6199a 2024-11-10T03:16:27,018 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 77ed36df592326523261438e65e5b722/B of 77ed36df592326523261438e65e5b722 into bddf2f14f2d24a3ea6fa9994e7e6199a(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:16:27,018 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 77ed36df592326523261438e65e5b722: 2024-11-10T03:16:27,018 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., storeName=77ed36df592326523261438e65e5b722/B, priority=12, startTime=1731208586822; duration=0sec 2024-11-10T03:16:27,018 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T03:16:27,018 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 77ed36df592326523261438e65e5b722:B 2024-11-10T03:16:27,018 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T03:16:27,020 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49430 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T03:16:27,020 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1541): 77ed36df592326523261438e65e5b722/C is initiating minor compaction (all files) 2024-11-10T03:16:27,020 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 77ed36df592326523261438e65e5b722/C in TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:27,021 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/18b79e975d43438d9ed669fbf4c38c3b, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/3f00d2151e6c417c9815126b19d2de00, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/64c4f6aee6f54bb9867f3eecfa40cdcb, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/c05ee47056fc4a1aab3294f34bf6271e] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp, totalSize=48.3 K 2024-11-10T03:16:27,022 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 18b79e975d43438d9ed669fbf4c38c3b, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731208581709 2024-11-10T03:16:27,023 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3f00d2151e6c417c9815126b19d2de00, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731208583685 2024-11-10T03:16:27,023 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 77ed36df592326523261438e65e5b722/A of 77ed36df592326523261438e65e5b722 into 905a50fe0e4945c580f1203d43d41c3f(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:16:27,023 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 77ed36df592326523261438e65e5b722: 2024-11-10T03:16:27,023 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 64c4f6aee6f54bb9867f3eecfa40cdcb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1731208585076 2024-11-10T03:16:27,023 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., storeName=77ed36df592326523261438e65e5b722/A, priority=12, startTime=1731208586821; duration=0sec 2024-11-10T03:16:27,024 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:16:27,024 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 77ed36df592326523261438e65e5b722:A 2024-11-10T03:16:27,024 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting c05ee47056fc4a1aab3294f34bf6271e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731208585768 2024-11-10T03:16:27,055 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 77ed36df592326523261438e65e5b722#C#compaction#48 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:16:27,056 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. as already flushing 2024-11-10T03:16:27,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on 77ed36df592326523261438e65e5b722 2024-11-10T03:16:27,056 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/C/834cd659a4924f78a0fafc8f0f89c777 is 50, key is test_row_0/C:col10/1731208585774/Put/seqid=0 2024-11-10T03:16:27,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741884_1060 (size=12301) 2024-11-10T03:16:27,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741885_1061 (size=12813) 2024-11-10T03:16:27,109 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:27,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51370 deadline: 1731208647097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:27,111 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:27,111 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:27,111 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:27,117 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:27,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51378 deadline: 1731208647110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:27,119 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:27,119 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:27,119 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:27,122 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:27,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51402 deadline: 1731208647111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:27,123 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:27,123 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:27,123 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:27,214 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:27,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51370 deadline: 1731208647213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:27,218 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:27,218 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:27,218 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:27,229 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:27,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51378 deadline: 1731208647222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:27,230 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:27,230 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:27,230 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:27,239 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:27,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51402 deadline: 1731208647233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:27,240 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:27,240 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:27,240 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:27,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:27,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51354 deadline: 1731208647282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:27,286 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 394d2222f724,38139,1731208572054 for 'test_row_1' in 77ed36df592326523261438e65e5b722 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8259 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T03:16:27,286 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:27,286 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:27,286 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:27,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-10T03:16:27,424 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:27,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51370 deadline: 1731208647423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:27,425 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:27,425 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:27,425 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:27,439 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:27,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51378 deadline: 1731208647433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:27,439 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:27,439 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:27,439 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:27,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:27,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51402 deadline: 1731208647443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:27,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:27,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:27,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:27,458 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/A/523e5f6c57814d5490dbe2d0e58cf575 2024-11-10T03:16:27,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/B/71f31734b5b14fe7ba1b946619380fad is 50, key is test_row_0/B:col10/1731208585901/Put/seqid=0 2024-11-10T03:16:27,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741886_1062 (size=12301) 2024-11-10T03:16:27,512 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/C/834cd659a4924f78a0fafc8f0f89c777 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/834cd659a4924f78a0fafc8f0f89c777 2024-11-10T03:16:27,523 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 77ed36df592326523261438e65e5b722/C of 77ed36df592326523261438e65e5b722 into 834cd659a4924f78a0fafc8f0f89c777(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:16:27,523 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 77ed36df592326523261438e65e5b722: 2024-11-10T03:16:27,523 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., storeName=77ed36df592326523261438e65e5b722/C, priority=12, startTime=1731208586822; duration=0sec 2024-11-10T03:16:27,523 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:16:27,523 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 77ed36df592326523261438e65e5b722:C 2024-11-10T03:16:27,734 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:27,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51370 deadline: 1731208647733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:27,736 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:27,736 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:27,736 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:27,749 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:27,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51378 deadline: 1731208647743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:27,749 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:27,750 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:27,750 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:27,756 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:27,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51402 deadline: 1731208647755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:27,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:27,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:27,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:27,894 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/B/71f31734b5b14fe7ba1b946619380fad 2024-11-10T03:16:27,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/C/83b07bfbc4424aecaa19b9afd9ad17b6 is 50, key is test_row_0/C:col10/1731208585901/Put/seqid=0 2024-11-10T03:16:27,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741887_1063 (size=12301) 2024-11-10T03:16:27,935 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/C/83b07bfbc4424aecaa19b9afd9ad17b6 2024-11-10T03:16:27,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/A/523e5f6c57814d5490dbe2d0e58cf575 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/523e5f6c57814d5490dbe2d0e58cf575 2024-11-10T03:16:27,956 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/523e5f6c57814d5490dbe2d0e58cf575, entries=150, sequenceid=364, filesize=12.0 K 2024-11-10T03:16:27,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/B/71f31734b5b14fe7ba1b946619380fad as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/71f31734b5b14fe7ba1b946619380fad 2024-11-10T03:16:27,976 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/71f31734b5b14fe7ba1b946619380fad, entries=150, sequenceid=364, filesize=12.0 K 2024-11-10T03:16:27,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/C/83b07bfbc4424aecaa19b9afd9ad17b6 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/83b07bfbc4424aecaa19b9afd9ad17b6 2024-11-10T03:16:27,985 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/83b07bfbc4424aecaa19b9afd9ad17b6, entries=150, sequenceid=364, filesize=12.0 K 2024-11-10T03:16:27,987 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~136.42 KB/139690, heapSize ~385.33 KB/394576, currentSize=53.67 KB/54960 for 77ed36df592326523261438e65e5b722 in 1014ms, sequenceid=364, compaction requested=false 2024-11-10T03:16:27,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 77ed36df592326523261438e65e5b722: 2024-11-10T03:16:27,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:27,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-10T03:16:27,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-10T03:16:27,996 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-10T03:16:27,997 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8250 sec 2024-11-10T03:16:28,013 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.8520 sec 2024-11-10T03:16:28,248 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 77ed36df592326523261438e65e5b722 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-10T03:16:28,249 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77ed36df592326523261438e65e5b722, store=A 2024-11-10T03:16:28,249 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:16:28,249 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77ed36df592326523261438e65e5b722, store=B 2024-11-10T03:16:28,249 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:16:28,249 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77ed36df592326523261438e65e5b722, store=C 2024-11-10T03:16:28,249 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:16:28,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on 77ed36df592326523261438e65e5b722 2024-11-10T03:16:28,267 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/A/f0846beea3b84d4891e5da0afd8439f7 is 50, key is test_row_0/A:col10/1731208587107/Put/seqid=0 2024-11-10T03:16:28,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-10T03:16:28,314 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T03:16:28,315 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T03:16:28,316 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:28,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51370 deadline: 1731208648307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:28,317 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:28,317 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:28,317 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:28,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741888_1064 (size=14741) 2024-11-10T03:16:28,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-10T03:16:28,326 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/A/f0846beea3b84d4891e5da0afd8439f7 2024-11-10T03:16:28,327 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T03:16:28,327 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:28,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51402 deadline: 1731208648318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:28,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:28,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:28,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:28,332 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T03:16:28,332 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T03:16:28,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-10T03:16:28,342 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:28,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51378 deadline: 1731208648333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:28,342 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/B/5341143ae1c3426aa28c71e4527585c3 is 50, key is test_row_0/B:col10/1731208587107/Put/seqid=0 2024-11-10T03:16:28,342 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:28,342 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:28,342 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:28,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741889_1065 (size=12301) 2024-11-10T03:16:28,432 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:28,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51370 deadline: 1731208648423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:28,432 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:28,433 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:28,433 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:28,434 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:28,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51402 deadline: 1731208648433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:28,435 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:28,435 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:28,435 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:28,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-10T03:16:28,454 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:28,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51378 deadline: 1731208648453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:28,454 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:28,455 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:28,455 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:28,488 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-10T03:16:28,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:28,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. as already flushing 2024-11-10T03:16:28,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:28,488 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:28,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:28,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:28,644 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-10T03:16:28,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:28,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. as already flushing 2024-11-10T03:16:28,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:28,645 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:28,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:28,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:28,646 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:28,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51370 deadline: 1731208648643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:28,647 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:28,647 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:28,647 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:28,650 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:28,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51402 deadline: 1731208648645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:28,651 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:28,651 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:28,651 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:28,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-10T03:16:28,671 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:28,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51378 deadline: 1731208648663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:28,672 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:28,672 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:28,672 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:28,759 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/B/5341143ae1c3426aa28c71e4527585c3 2024-11-10T03:16:28,776 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/C/186e0006b1cb4782bdeae8fdc44d0e18 is 50, key is test_row_0/C:col10/1731208587107/Put/seqid=0 2024-11-10T03:16:28,798 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-10T03:16:28,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:28,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. as already flushing 2024-11-10T03:16:28,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:28,798 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:28,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:28,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:28,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741890_1066 (size=12301) 2024-11-10T03:16:28,822 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/C/186e0006b1cb4782bdeae8fdc44d0e18 2024-11-10T03:16:28,836 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/A/f0846beea3b84d4891e5da0afd8439f7 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/f0846beea3b84d4891e5da0afd8439f7 2024-11-10T03:16:28,856 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/f0846beea3b84d4891e5da0afd8439f7, entries=200, sequenceid=379, filesize=14.4 K 2024-11-10T03:16:28,858 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/B/5341143ae1c3426aa28c71e4527585c3 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/5341143ae1c3426aa28c71e4527585c3 2024-11-10T03:16:28,869 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/5341143ae1c3426aa28c71e4527585c3, entries=150, sequenceid=379, filesize=12.0 K 2024-11-10T03:16:28,877 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/C/186e0006b1cb4782bdeae8fdc44d0e18 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/186e0006b1cb4782bdeae8fdc44d0e18 2024-11-10T03:16:28,893 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/186e0006b1cb4782bdeae8fdc44d0e18, entries=150, sequenceid=379, filesize=12.0 K 2024-11-10T03:16:28,896 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 77ed36df592326523261438e65e5b722 in 647ms, sequenceid=379, compaction requested=true 2024-11-10T03:16:28,896 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 77ed36df592326523261438e65e5b722: 2024-11-10T03:16:28,897 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T03:16:28,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 77ed36df592326523261438e65e5b722:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T03:16:28,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:16:28,898 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T03:16:28,899 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 77ed36df592326523261438e65e5b722:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T03:16:28,899 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39855 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T03:16:28,899 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:16:28,899 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1541): 77ed36df592326523261438e65e5b722/A is initiating minor compaction (all files) 2024-11-10T03:16:28,899 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 77ed36df592326523261438e65e5b722:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T03:16:28,899 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 77ed36df592326523261438e65e5b722/A in TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:28,899 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T03:16:28,899 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/905a50fe0e4945c580f1203d43d41c3f, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/523e5f6c57814d5490dbe2d0e58cf575, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/f0846beea3b84d4891e5da0afd8439f7] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp, totalSize=38.9 K 2024-11-10T03:16:28,900 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37415 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T03:16:28,900 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1541): 77ed36df592326523261438e65e5b722/B is initiating minor compaction (all files) 2024-11-10T03:16:28,900 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 77ed36df592326523261438e65e5b722/B in TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:28,900 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/bddf2f14f2d24a3ea6fa9994e7e6199a, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/71f31734b5b14fe7ba1b946619380fad, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/5341143ae1c3426aa28c71e4527585c3] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp, totalSize=36.5 K 2024-11-10T03:16:28,900 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 905a50fe0e4945c580f1203d43d41c3f, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731208585768 2024-11-10T03:16:28,902 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 523e5f6c57814d5490dbe2d0e58cf575, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1731208585889 2024-11-10T03:16:28,903 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting bddf2f14f2d24a3ea6fa9994e7e6199a, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731208585768 2024-11-10T03:16:28,904 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 71f31734b5b14fe7ba1b946619380fad, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1731208585889 2024-11-10T03:16:28,904 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting f0846beea3b84d4891e5da0afd8439f7, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1731208587085 2024-11-10T03:16:28,907 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 5341143ae1c3426aa28c71e4527585c3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1731208587085 2024-11-10T03:16:28,930 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 77ed36df592326523261438e65e5b722#A#compaction#55 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:16:28,930 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/A/4d15eb855984493994e110fdda9a2954 is 50, key is test_row_0/A:col10/1731208587107/Put/seqid=0 2024-11-10T03:16:28,931 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 77ed36df592326523261438e65e5b722#B#compaction#54 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:16:28,931 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/B/ed4f506be2e849638e5e2c1501cd0d40 is 50, key is test_row_0/B:col10/1731208587107/Put/seqid=0 2024-11-10T03:16:28,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741892_1068 (size=12915) 2024-11-10T03:16:28,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741891_1067 (size=12915) 2024-11-10T03:16:28,954 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-10T03:16:28,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:28,954 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 77ed36df592326523261438e65e5b722 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-10T03:16:28,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77ed36df592326523261438e65e5b722, store=A 2024-11-10T03:16:28,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:16:28,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77ed36df592326523261438e65e5b722, store=B 2024-11-10T03:16:28,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:16:28,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77ed36df592326523261438e65e5b722, store=C 2024-11-10T03:16:28,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:16:28,957 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/B/ed4f506be2e849638e5e2c1501cd0d40 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/ed4f506be2e849638e5e2c1501cd0d40 2024-11-10T03:16:28,959 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. as already flushing 2024-11-10T03:16:28,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on 77ed36df592326523261438e65e5b722 2024-11-10T03:16:28,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-10T03:16:28,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/A/b2f5afa9366949f8bd146386f28b2ece is 50, key is test_row_0/A:col10/1731208588295/Put/seqid=0 2024-11-10T03:16:28,977 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 77ed36df592326523261438e65e5b722/B of 77ed36df592326523261438e65e5b722 into ed4f506be2e849638e5e2c1501cd0d40(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:16:28,977 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 77ed36df592326523261438e65e5b722: 2024-11-10T03:16:28,978 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., storeName=77ed36df592326523261438e65e5b722/B, priority=13, startTime=1731208588898; duration=0sec 2024-11-10T03:16:28,978 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T03:16:28,978 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 77ed36df592326523261438e65e5b722:B 2024-11-10T03:16:28,978 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T03:16:28,984 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37415 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T03:16:28,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741893_1069 (size=12301) 2024-11-10T03:16:28,984 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1541): 77ed36df592326523261438e65e5b722/C is initiating minor compaction (all files) 2024-11-10T03:16:28,984 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 77ed36df592326523261438e65e5b722/C in TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:28,984 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/834cd659a4924f78a0fafc8f0f89c777, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/83b07bfbc4424aecaa19b9afd9ad17b6, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/186e0006b1cb4782bdeae8fdc44d0e18] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp, totalSize=36.5 K 2024-11-10T03:16:28,986 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 834cd659a4924f78a0fafc8f0f89c777, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731208585768 2024-11-10T03:16:28,986 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 83b07bfbc4424aecaa19b9afd9ad17b6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1731208585889 2024-11-10T03:16:28,989 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 186e0006b1cb4782bdeae8fdc44d0e18, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1731208587085 2024-11-10T03:16:29,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:29,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51370 deadline: 1731208648993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:29,002 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:29,002 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:29,002 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:29,006 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:29,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51378 deadline: 1731208649003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:29,009 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 77ed36df592326523261438e65e5b722#C#compaction#57 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:16:29,009 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:29,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51402 deadline: 1731208649003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:29,010 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/C/e7797116da52449fa8540dc9dbdfda81 is 50, key is test_row_0/C:col10/1731208587107/Put/seqid=0 2024-11-10T03:16:29,010 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:29,010 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:29,010 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:29,011 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:29,011 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:29,011 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:29,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741894_1070 (size=12915) 2024-11-10T03:16:29,113 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:29,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51378 deadline: 1731208649113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:29,114 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:29,114 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:29,114 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:29,114 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:29,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51370 deadline: 1731208649113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:29,115 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:29,115 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:29,115 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:29,115 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:29,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51402 deadline: 1731208649113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:29,116 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:29,116 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:29,116 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:29,331 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:29,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51370 deadline: 1731208649325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:29,330 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:29,336 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:29,336 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:29,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51378 deadline: 1731208649325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:29,336 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:29,337 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:29,337 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:29,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51402 deadline: 1731208649325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:29,337 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:29,337 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:29,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:29,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:29,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:29,359 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/A/4d15eb855984493994e110fdda9a2954 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/4d15eb855984493994e110fdda9a2954 2024-11-10T03:16:29,370 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 77ed36df592326523261438e65e5b722/A of 77ed36df592326523261438e65e5b722 into 4d15eb855984493994e110fdda9a2954(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:16:29,371 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 77ed36df592326523261438e65e5b722: 2024-11-10T03:16:29,371 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., storeName=77ed36df592326523261438e65e5b722/A, priority=13, startTime=1731208588897; duration=0sec 2024-11-10T03:16:29,371 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:16:29,371 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 77ed36df592326523261438e65e5b722:A 2024-11-10T03:16:29,386 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=399 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/A/b2f5afa9366949f8bd146386f28b2ece 2024-11-10T03:16:29,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/B/50195dcec2904f42833edd14a5f180ad is 50, key is test_row_0/B:col10/1731208588295/Put/seqid=0 2024-11-10T03:16:29,441 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/C/e7797116da52449fa8540dc9dbdfda81 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/e7797116da52449fa8540dc9dbdfda81 2024-11-10T03:16:29,451 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 77ed36df592326523261438e65e5b722/C of 77ed36df592326523261438e65e5b722 into e7797116da52449fa8540dc9dbdfda81(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:16:29,451 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 77ed36df592326523261438e65e5b722: 2024-11-10T03:16:29,451 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., storeName=77ed36df592326523261438e65e5b722/C, priority=13, startTime=1731208588899; duration=0sec 2024-11-10T03:16:29,451 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:16:29,451 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 77ed36df592326523261438e65e5b722:C 2024-11-10T03:16:29,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741895_1071 (size=12301) 2024-11-10T03:16:29,459 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=399 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/B/50195dcec2904f42833edd14a5f180ad 2024-11-10T03:16:29,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-10T03:16:29,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/C/04ea8793b81e4692bc6d285de64e07c9 is 50, key is test_row_0/C:col10/1731208588295/Put/seqid=0 2024-11-10T03:16:29,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741896_1072 (size=12301) 2024-11-10T03:16:29,644 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:29,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51370 deadline: 1731208649643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:29,645 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:29,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51378 deadline: 1731208649643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:29,645 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:29,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51402 deadline: 1731208649643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:29,646 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:29,646 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:29,646 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:29,646 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:29,646 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:29,646 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:29,646 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:29,646 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:29,646 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:29,915 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=399 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/C/04ea8793b81e4692bc6d285de64e07c9 2024-11-10T03:16:29,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/A/b2f5afa9366949f8bd146386f28b2ece as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/b2f5afa9366949f8bd146386f28b2ece 2024-11-10T03:16:29,931 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/b2f5afa9366949f8bd146386f28b2ece, entries=150, sequenceid=399, filesize=12.0 K 2024-11-10T03:16:29,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/B/50195dcec2904f42833edd14a5f180ad as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/50195dcec2904f42833edd14a5f180ad 2024-11-10T03:16:29,945 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/50195dcec2904f42833edd14a5f180ad, entries=150, sequenceid=399, filesize=12.0 K 2024-11-10T03:16:29,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/C/04ea8793b81e4692bc6d285de64e07c9 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/04ea8793b81e4692bc6d285de64e07c9 2024-11-10T03:16:29,953 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/04ea8793b81e4692bc6d285de64e07c9, entries=150, sequenceid=399, filesize=12.0 K 2024-11-10T03:16:29,954 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for 77ed36df592326523261438e65e5b722 in 1000ms, sequenceid=399, compaction requested=false 2024-11-10T03:16:29,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 77ed36df592326523261438e65e5b722: 2024-11-10T03:16:29,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:29,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-10T03:16:29,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-10T03:16:29,958 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-10T03:16:29,958 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6240 sec 2024-11-10T03:16:29,963 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 1.6440 sec 2024-11-10T03:16:30,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on 77ed36df592326523261438e65e5b722 2024-11-10T03:16:30,006 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 77ed36df592326523261438e65e5b722 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-10T03:16:30,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77ed36df592326523261438e65e5b722, store=A 2024-11-10T03:16:30,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:16:30,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77ed36df592326523261438e65e5b722, store=B 2024-11-10T03:16:30,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:16:30,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77ed36df592326523261438e65e5b722, store=C 2024-11-10T03:16:30,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:16:30,012 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/A/d263d8ba23164bee8d31995ddae0034e is 50, key is test_row_0/A:col10/1731208588990/Put/seqid=0 2024-11-10T03:16:30,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741897_1073 (size=12301) 2024-11-10T03:16:30,023 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/A/d263d8ba23164bee8d31995ddae0034e 2024-11-10T03:16:30,038 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/B/38a7848ea2a64fe8951b74af9b96f014 is 50, key is test_row_0/B:col10/1731208588990/Put/seqid=0 2024-11-10T03:16:30,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741898_1074 (size=12301) 2024-11-10T03:16:30,051 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/B/38a7848ea2a64fe8951b74af9b96f014 2024-11-10T03:16:30,062 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/C/67c5ef171591440198f81869accd5790 is 50, key is test_row_0/C:col10/1731208588990/Put/seqid=0 2024-11-10T03:16:30,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741899_1075 (size=12301) 2024-11-10T03:16:30,071 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/C/67c5ef171591440198f81869accd5790 2024-11-10T03:16:30,080 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/A/d263d8ba23164bee8d31995ddae0034e as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/d263d8ba23164bee8d31995ddae0034e 2024-11-10T03:16:30,080 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:30,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51380 deadline: 1731208650076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:30,081 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:30,081 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:30,081 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:30,086 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/d263d8ba23164bee8d31995ddae0034e, entries=150, sequenceid=416, filesize=12.0 K 2024-11-10T03:16:30,088 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/B/38a7848ea2a64fe8951b74af9b96f014 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/38a7848ea2a64fe8951b74af9b96f014 2024-11-10T03:16:30,094 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/38a7848ea2a64fe8951b74af9b96f014, entries=150, sequenceid=416, filesize=12.0 K 2024-11-10T03:16:30,095 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/C/67c5ef171591440198f81869accd5790 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/67c5ef171591440198f81869accd5790 2024-11-10T03:16:30,104 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/67c5ef171591440198f81869accd5790, entries=150, sequenceid=416, filesize=12.0 K 2024-11-10T03:16:30,105 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 77ed36df592326523261438e65e5b722 in 100ms, sequenceid=416, compaction requested=true 2024-11-10T03:16:30,105 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 77ed36df592326523261438e65e5b722: 2024-11-10T03:16:30,105 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 77ed36df592326523261438e65e5b722:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T03:16:30,105 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:16:30,105 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T03:16:30,105 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 77ed36df592326523261438e65e5b722:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T03:16:30,105 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T03:16:30,105 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:16:30,105 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 77ed36df592326523261438e65e5b722:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T03:16:30,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T03:16:30,106 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T03:16:30,106 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T03:16:30,106 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1541): 77ed36df592326523261438e65e5b722/B is initiating minor compaction (all files) 2024-11-10T03:16:30,107 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1541): 77ed36df592326523261438e65e5b722/A is initiating minor compaction (all files) 2024-11-10T03:16:30,107 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 77ed36df592326523261438e65e5b722/B in TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:30,107 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 77ed36df592326523261438e65e5b722/A in TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:30,107 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/4d15eb855984493994e110fdda9a2954, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/b2f5afa9366949f8bd146386f28b2ece, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/d263d8ba23164bee8d31995ddae0034e] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp, totalSize=36.6 K 2024-11-10T03:16:30,107 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/ed4f506be2e849638e5e2c1501cd0d40, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/50195dcec2904f42833edd14a5f180ad, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/38a7848ea2a64fe8951b74af9b96f014] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp, totalSize=36.6 K 2024-11-10T03:16:30,108 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting ed4f506be2e849638e5e2c1501cd0d40, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1731208587085 2024-11-10T03:16:30,109 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 50195dcec2904f42833edd14a5f180ad, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1731208588295 2024-11-10T03:16:30,109 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d15eb855984493994e110fdda9a2954, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1731208587085 2024-11-10T03:16:30,109 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 38a7848ea2a64fe8951b74af9b96f014, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1731208588988 2024-11-10T03:16:30,109 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting b2f5afa9366949f8bd146386f28b2ece, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1731208588295 2024-11-10T03:16:30,110 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting d263d8ba23164bee8d31995ddae0034e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1731208588988 2024-11-10T03:16:30,126 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 77ed36df592326523261438e65e5b722#A#compaction#63 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:16:30,127 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/A/558ad750b42646beade61cee98057759 is 50, key is test_row_0/A:col10/1731208588990/Put/seqid=0 2024-11-10T03:16:30,128 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 77ed36df592326523261438e65e5b722#B#compaction#64 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:16:30,129 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/B/4c1ecc6e66a3480391ba74e668ee873d is 50, key is test_row_0/B:col10/1731208588990/Put/seqid=0 2024-11-10T03:16:30,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741900_1076 (size=13017) 2024-11-10T03:16:30,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741901_1077 (size=13017) 2024-11-10T03:16:30,150 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/A/558ad750b42646beade61cee98057759 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/558ad750b42646beade61cee98057759 2024-11-10T03:16:30,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on 77ed36df592326523261438e65e5b722 2024-11-10T03:16:30,159 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 77ed36df592326523261438e65e5b722 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-10T03:16:30,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77ed36df592326523261438e65e5b722, store=A 2024-11-10T03:16:30,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:16:30,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77ed36df592326523261438e65e5b722, store=B 2024-11-10T03:16:30,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:16:30,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77ed36df592326523261438e65e5b722, store=C 2024-11-10T03:16:30,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:16:30,164 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/B/4c1ecc6e66a3480391ba74e668ee873d as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/4c1ecc6e66a3480391ba74e668ee873d 2024-11-10T03:16:30,169 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 77ed36df592326523261438e65e5b722/A of 77ed36df592326523261438e65e5b722 into 558ad750b42646beade61cee98057759(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:16:30,169 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 77ed36df592326523261438e65e5b722: 2024-11-10T03:16:30,169 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., storeName=77ed36df592326523261438e65e5b722/A, priority=13, startTime=1731208590105; duration=0sec 2024-11-10T03:16:30,169 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T03:16:30,169 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 77ed36df592326523261438e65e5b722:A 2024-11-10T03:16:30,169 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T03:16:30,170 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/A/c71a699f5dc642ddaaff60c8e1623c27 is 50, key is test_row_0/A:col10/1731208590157/Put/seqid=0 2024-11-10T03:16:30,178 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 77ed36df592326523261438e65e5b722/B of 77ed36df592326523261438e65e5b722 into 4c1ecc6e66a3480391ba74e668ee873d(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:16:30,178 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 77ed36df592326523261438e65e5b722: 2024-11-10T03:16:30,178 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., storeName=77ed36df592326523261438e65e5b722/B, priority=13, startTime=1731208590105; duration=0sec 2024-11-10T03:16:30,178 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:16:30,178 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 77ed36df592326523261438e65e5b722:B 2024-11-10T03:16:30,180 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T03:16:30,180 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1541): 77ed36df592326523261438e65e5b722/C is initiating minor compaction (all files) 2024-11-10T03:16:30,180 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 77ed36df592326523261438e65e5b722/C in TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:30,181 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/e7797116da52449fa8540dc9dbdfda81, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/04ea8793b81e4692bc6d285de64e07c9, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/67c5ef171591440198f81869accd5790] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp, totalSize=36.6 K 2024-11-10T03:16:30,181 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting e7797116da52449fa8540dc9dbdfda81, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1731208587085 2024-11-10T03:16:30,182 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 04ea8793b81e4692bc6d285de64e07c9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1731208588295 2024-11-10T03:16:30,183 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 67c5ef171591440198f81869accd5790, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1731208588988 2024-11-10T03:16:30,196 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:30,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51378 deadline: 1731208650188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:30,197 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:30,197 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:30,197 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:30,200 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:30,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51380 deadline: 1731208650194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:30,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:30,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51370 deadline: 1731208650197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:30,200 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:30,200 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:30,200 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:30,201 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 77ed36df592326523261438e65e5b722#C#compaction#66 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:16:30,201 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:30,201 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:30,201 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/C/384fc2a7ac634580ba937629f9a7f372 is 50, key is test_row_0/C:col10/1731208588990/Put/seqid=0 2024-11-10T03:16:30,201 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:30,205 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:30,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51402 deadline: 1731208650201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:30,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:30,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:30,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:30,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741902_1078 (size=12301) 2024-11-10T03:16:30,219 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=436 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/A/c71a699f5dc642ddaaff60c8e1623c27 2024-11-10T03:16:30,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741903_1079 (size=13017) 2024-11-10T03:16:30,242 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/B/5f62f3d4e99e4491a40b85e0ba9a7f37 is 50, key is test_row_0/B:col10/1731208590157/Put/seqid=0 2024-11-10T03:16:30,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741904_1080 (size=12301) 2024-11-10T03:16:30,254 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=436 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/B/5f62f3d4e99e4491a40b85e0ba9a7f37 2024-11-10T03:16:30,277 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/C/0daba8dbe18446b38f257f6d44a30d2d is 50, key is test_row_0/C:col10/1731208590157/Put/seqid=0 2024-11-10T03:16:30,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741905_1081 (size=12301) 2024-11-10T03:16:30,303 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=436 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/C/0daba8dbe18446b38f257f6d44a30d2d 2024-11-10T03:16:30,310 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/A/c71a699f5dc642ddaaff60c8e1623c27 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/c71a699f5dc642ddaaff60c8e1623c27 2024-11-10T03:16:30,313 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:30,313 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:30,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51378 deadline: 1731208650303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:30,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51380 deadline: 1731208650303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:30,314 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:30,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51370 deadline: 1731208650303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:30,314 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:30,314 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:30,314 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:30,314 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:30,314 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:30,314 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:30,315 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:30,315 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:30,315 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:30,316 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:30,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51402 deadline: 1731208650313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:30,317 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:30,317 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:30,317 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:30,321 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/c71a699f5dc642ddaaff60c8e1623c27, entries=150, sequenceid=436, filesize=12.0 K 2024-11-10T03:16:30,323 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/B/5f62f3d4e99e4491a40b85e0ba9a7f37 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/5f62f3d4e99e4491a40b85e0ba9a7f37 2024-11-10T03:16:30,329 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/5f62f3d4e99e4491a40b85e0ba9a7f37, entries=150, sequenceid=436, filesize=12.0 K 2024-11-10T03:16:30,331 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/C/0daba8dbe18446b38f257f6d44a30d2d as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/0daba8dbe18446b38f257f6d44a30d2d 2024-11-10T03:16:30,343 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/0daba8dbe18446b38f257f6d44a30d2d, entries=150, sequenceid=436, filesize=12.0 K 2024-11-10T03:16:30,344 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for 77ed36df592326523261438e65e5b722 in 185ms, sequenceid=436, compaction requested=false 2024-11-10T03:16:30,344 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 77ed36df592326523261438e65e5b722: 2024-11-10T03:16:30,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-10T03:16:30,483 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T03:16:30,484 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T03:16:30,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-10T03:16:30,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-10T03:16:30,486 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T03:16:30,487 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T03:16:30,487 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T03:16:30,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on 77ed36df592326523261438e65e5b722 2024-11-10T03:16:30,524 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 77ed36df592326523261438e65e5b722 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-10T03:16:30,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77ed36df592326523261438e65e5b722, store=A 2024-11-10T03:16:30,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:16:30,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77ed36df592326523261438e65e5b722, store=B 2024-11-10T03:16:30,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:16:30,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77ed36df592326523261438e65e5b722, store=C 2024-11-10T03:16:30,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:16:30,536 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/A/5e6420f527874f4a9ec60d977680e4c8 is 50, key is test_row_0/A:col10/1731208590523/Put/seqid=0 2024-11-10T03:16:30,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741906_1082 (size=14741) 2024-11-10T03:16:30,565 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:30,565 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:30,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51402 deadline: 1731208650552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:30,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51378 deadline: 1731208650552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:30,566 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:30,566 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:30,566 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:30,566 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:30,566 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:30,566 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:30,568 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:30,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51370 deadline: 1731208650566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:30,569 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:30,569 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:30,569 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:30,571 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:30,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51380 deadline: 1731208650567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:30,572 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:30,572 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:30,572 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:30,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-10T03:16:30,639 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-10T03:16:30,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:30,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. as already flushing 2024-11-10T03:16:30,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:30,641 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:30,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:30,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:30,644 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/C/384fc2a7ac634580ba937629f9a7f372 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/384fc2a7ac634580ba937629f9a7f372 2024-11-10T03:16:30,654 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 77ed36df592326523261438e65e5b722/C of 77ed36df592326523261438e65e5b722 into 384fc2a7ac634580ba937629f9a7f372(size=12.7 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:16:30,654 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 77ed36df592326523261438e65e5b722: 2024-11-10T03:16:30,654 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., storeName=77ed36df592326523261438e65e5b722/C, priority=13, startTime=1731208590105; duration=0sec 2024-11-10T03:16:30,654 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:16:30,654 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 77ed36df592326523261438e65e5b722:C 2024-11-10T03:16:30,689 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:30,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51402 deadline: 1731208650684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:30,690 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:30,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51370 deadline: 1731208650684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:30,690 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:30,690 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:30,690 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:30,690 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:30,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51380 deadline: 1731208650685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:30,690 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:30,690 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:30,690 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:30,690 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:30,690 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:30,690 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:30,690 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:30,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51378 deadline: 1731208650685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:30,691 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:30,691 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:30,691 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:30,796 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-10T03:16:30,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:30,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. as already flushing 2024-11-10T03:16:30,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:30,796 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:30,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:30,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:30,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-10T03:16:30,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:30,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51402 deadline: 1731208650893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:30,897 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:30,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51370 deadline: 1731208650893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:30,897 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:30,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:30,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:30,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51380 deadline: 1731208650893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:30,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:30,897 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:30,897 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:30,898 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:30,898 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:30,898 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:30,898 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:30,903 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:30,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51378 deadline: 1731208650903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:30,905 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:30,905 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:30,905 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:30,941 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=452 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/A/5e6420f527874f4a9ec60d977680e4c8 2024-11-10T03:16:30,951 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-10T03:16:30,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:30,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. as already flushing 2024-11-10T03:16:30,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:30,951 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:30,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:30,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:30,954 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/B/85a2ab167dbf4767bc1bc36fd43ea015 is 50, key is test_row_0/B:col10/1731208590523/Put/seqid=0 2024-11-10T03:16:30,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741907_1083 (size=12301) 2024-11-10T03:16:31,104 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-10T03:16:31,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:31,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. as already flushing 2024-11-10T03:16:31,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:31,105 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:31,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:31,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:31,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-10T03:16:31,203 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:31,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51402 deadline: 1731208651203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:31,204 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:31,204 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:31,204 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:31,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51380 deadline: 1731208651203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:31,204 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:31,204 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:31,204 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:31,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51370 deadline: 1731208651203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:31,204 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:31,204 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:31,205 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:31,205 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:31,205 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:31,213 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:31,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51378 deadline: 1731208651213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:31,214 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:31,214 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:31,214 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:31,259 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-10T03:16:31,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:31,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. as already flushing 2024-11-10T03:16:31,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:31,259 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:31,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:31,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:31,375 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=452 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/B/85a2ab167dbf4767bc1bc36fd43ea015 2024-11-10T03:16:31,383 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/C/0306fb6329ab45e7a5c4f0a881dd3055 is 50, key is test_row_0/C:col10/1731208590523/Put/seqid=0 2024-11-10T03:16:31,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741908_1084 (size=12301) 2024-11-10T03:16:31,388 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=452 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/C/0306fb6329ab45e7a5c4f0a881dd3055 2024-11-10T03:16:31,394 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/A/5e6420f527874f4a9ec60d977680e4c8 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/5e6420f527874f4a9ec60d977680e4c8 2024-11-10T03:16:31,401 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/5e6420f527874f4a9ec60d977680e4c8, entries=200, sequenceid=452, filesize=14.4 K 2024-11-10T03:16:31,402 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/B/85a2ab167dbf4767bc1bc36fd43ea015 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/85a2ab167dbf4767bc1bc36fd43ea015 2024-11-10T03:16:31,407 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/85a2ab167dbf4767bc1bc36fd43ea015, entries=150, sequenceid=452, filesize=12.0 K 2024-11-10T03:16:31,408 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/C/0306fb6329ab45e7a5c4f0a881dd3055 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/0306fb6329ab45e7a5c4f0a881dd3055 2024-11-10T03:16:31,413 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-10T03:16:31,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:31,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. as already flushing 2024-11-10T03:16:31,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:31,413 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:31,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:31,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:31,416 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/0306fb6329ab45e7a5c4f0a881dd3055, entries=150, sequenceid=452, filesize=12.0 K 2024-11-10T03:16:31,417 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for 77ed36df592326523261438e65e5b722 in 893ms, sequenceid=452, compaction requested=true 2024-11-10T03:16:31,417 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 77ed36df592326523261438e65e5b722: 2024-11-10T03:16:31,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 77ed36df592326523261438e65e5b722:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T03:16:31,418 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T03:16:31,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:16:31,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 77ed36df592326523261438e65e5b722:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T03:16:31,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:16:31,418 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T03:16:31,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 77ed36df592326523261438e65e5b722:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T03:16:31,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T03:16:31,419 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40059 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T03:16:31,419 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T03:16:31,419 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1541): 77ed36df592326523261438e65e5b722/A is initiating minor compaction (all files) 2024-11-10T03:16:31,419 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1541): 77ed36df592326523261438e65e5b722/B is initiating minor compaction (all files) 2024-11-10T03:16:31,419 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 77ed36df592326523261438e65e5b722/A in TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:31,419 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 77ed36df592326523261438e65e5b722/B in TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:31,419 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/4c1ecc6e66a3480391ba74e668ee873d, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/5f62f3d4e99e4491a40b85e0ba9a7f37, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/85a2ab167dbf4767bc1bc36fd43ea015] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp, totalSize=36.7 K 2024-11-10T03:16:31,419 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/558ad750b42646beade61cee98057759, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/c71a699f5dc642ddaaff60c8e1623c27, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/5e6420f527874f4a9ec60d977680e4c8] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp, totalSize=39.1 K 2024-11-10T03:16:31,419 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 558ad750b42646beade61cee98057759, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1731208588988 2024-11-10T03:16:31,419 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c1ecc6e66a3480391ba74e668ee873d, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1731208588988 2024-11-10T03:16:31,420 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting c71a699f5dc642ddaaff60c8e1623c27, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=436, earliestPutTs=1731208590072 2024-11-10T03:16:31,420 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f62f3d4e99e4491a40b85e0ba9a7f37, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=436, earliestPutTs=1731208590072 2024-11-10T03:16:31,421 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5e6420f527874f4a9ec60d977680e4c8, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=452, earliestPutTs=1731208590174 2024-11-10T03:16:31,421 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 85a2ab167dbf4767bc1bc36fd43ea015, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=452, earliestPutTs=1731208590174 2024-11-10T03:16:31,430 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 77ed36df592326523261438e65e5b722#B#compaction#72 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:16:31,431 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 77ed36df592326523261438e65e5b722#A#compaction#73 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:16:31,431 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/B/9a7c66c8aec8411493926b38ef391bc3 is 50, key is test_row_0/B:col10/1731208590523/Put/seqid=0 2024-11-10T03:16:31,431 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/A/412be7d500184f6898983229b07f9784 is 50, key is test_row_0/A:col10/1731208590523/Put/seqid=0 2024-11-10T03:16:31,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741909_1085 (size=13119) 2024-11-10T03:16:31,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741910_1086 (size=13119) 2024-11-10T03:16:31,463 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/A/412be7d500184f6898983229b07f9784 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/412be7d500184f6898983229b07f9784 2024-11-10T03:16:31,469 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 77ed36df592326523261438e65e5b722/A of 77ed36df592326523261438e65e5b722 into 412be7d500184f6898983229b07f9784(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:16:31,469 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 77ed36df592326523261438e65e5b722: 2024-11-10T03:16:31,469 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., storeName=77ed36df592326523261438e65e5b722/A, priority=13, startTime=1731208591417; duration=0sec 2024-11-10T03:16:31,469 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T03:16:31,469 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 77ed36df592326523261438e65e5b722:A 2024-11-10T03:16:31,469 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T03:16:31,471 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T03:16:31,471 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1541): 77ed36df592326523261438e65e5b722/C is initiating minor compaction (all files) 2024-11-10T03:16:31,471 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 77ed36df592326523261438e65e5b722/C in TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:31,471 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/384fc2a7ac634580ba937629f9a7f372, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/0daba8dbe18446b38f257f6d44a30d2d, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/0306fb6329ab45e7a5c4f0a881dd3055] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp, totalSize=36.7 K 2024-11-10T03:16:31,471 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 384fc2a7ac634580ba937629f9a7f372, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1731208588988 2024-11-10T03:16:31,472 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0daba8dbe18446b38f257f6d44a30d2d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=436, earliestPutTs=1731208590072 2024-11-10T03:16:31,472 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0306fb6329ab45e7a5c4f0a881dd3055, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=452, earliestPutTs=1731208590174 2024-11-10T03:16:31,483 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 77ed36df592326523261438e65e5b722#C#compaction#74 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:16:31,484 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/C/49c70c22bf1e4a15a27d5c8c7b7137ee is 50, key is test_row_0/C:col10/1731208590523/Put/seqid=0 2024-11-10T03:16:31,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741911_1087 (size=13119) 2024-11-10T03:16:31,499 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/C/49c70c22bf1e4a15a27d5c8c7b7137ee as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/49c70c22bf1e4a15a27d5c8c7b7137ee 2024-11-10T03:16:31,508 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 77ed36df592326523261438e65e5b722/C of 77ed36df592326523261438e65e5b722 into 49c70c22bf1e4a15a27d5c8c7b7137ee(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:16:31,508 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 77ed36df592326523261438e65e5b722: 2024-11-10T03:16:31,508 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., storeName=77ed36df592326523261438e65e5b722/C, priority=13, startTime=1731208591418; duration=0sec 2024-11-10T03:16:31,508 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:16:31,508 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 77ed36df592326523261438e65e5b722:C 2024-11-10T03:16:31,567 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-10T03:16:31,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:31,567 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 77ed36df592326523261438e65e5b722 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-10T03:16:31,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77ed36df592326523261438e65e5b722, store=A 2024-11-10T03:16:31,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:16:31,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77ed36df592326523261438e65e5b722, store=B 2024-11-10T03:16:31,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:16:31,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77ed36df592326523261438e65e5b722, store=C 2024-11-10T03:16:31,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:16:31,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/A/cfe4475b44ee406e809238739dc232b0 is 50, key is test_row_0/A:col10/1731208590563/Put/seqid=0 2024-11-10T03:16:31,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741912_1088 (size=12301) 2024-11-10T03:16:31,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-10T03:16:31,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on 77ed36df592326523261438e65e5b722 2024-11-10T03:16:31,716 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. as already flushing 2024-11-10T03:16:31,731 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:31,731 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:31,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51378 deadline: 1731208651727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:31,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51380 deadline: 1731208651730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:31,731 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:31,731 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:31,731 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:31,731 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:31,731 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:31,731 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:31,737 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:31,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51370 deadline: 1731208651731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:31,737 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:31,737 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:31,737 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:31,737 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:31,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51402 deadline: 1731208651731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:31,738 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:31,738 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:31,738 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:31,845 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:31,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51380 deadline: 1731208651843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:31,845 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:31,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51378 deadline: 1731208651843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:31,846 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:31,846 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:31,846 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:31,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51370 deadline: 1731208651844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:31,846 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:31,846 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:31,846 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:31,846 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:31,846 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:31,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51402 deadline: 1731208651844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:31,846 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:31,847 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:31,847 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:31,847 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:31,847 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:31,847 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:31,864 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/B/9a7c66c8aec8411493926b38ef391bc3 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/9a7c66c8aec8411493926b38ef391bc3 2024-11-10T03:16:31,876 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 77ed36df592326523261438e65e5b722/B of 77ed36df592326523261438e65e5b722 into 9a7c66c8aec8411493926b38ef391bc3(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:16:31,876 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 77ed36df592326523261438e65e5b722: 2024-11-10T03:16:31,876 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., storeName=77ed36df592326523261438e65e5b722/B, priority=13, startTime=1731208591418; duration=0sec 2024-11-10T03:16:31,877 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:16:31,877 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 77ed36df592326523261438e65e5b722:B 2024-11-10T03:16:31,987 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=474 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/A/cfe4475b44ee406e809238739dc232b0 2024-11-10T03:16:32,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/B/5028a17b18c144d7a38e4721f6db56cf is 50, key is test_row_0/B:col10/1731208590563/Put/seqid=0 2024-11-10T03:16:32,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741913_1089 (size=12301) 2024-11-10T03:16:32,061 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:32,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51402 deadline: 1731208652053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:32,062 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:32,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:32,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:32,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51380 deadline: 1731208652053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:32,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:32,062 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:32,062 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:32,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51378 deadline: 1731208652053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:32,063 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:32,063 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:32,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51370 deadline: 1731208652053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:32,063 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:32,063 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:32,063 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:32,063 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:32,063 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:32,063 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:32,063 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:32,380 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:32,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51378 deadline: 1731208652374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:32,381 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:32,381 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:32,381 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:32,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51402 deadline: 1731208652374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:32,381 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:32,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:32,381 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:32,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:32,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:32,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51370 deadline: 1731208652374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:32,382 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:32,382 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:32,382 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:32,382 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:32,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51380 deadline: 1731208652374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:32,382 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:32,382 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:32,383 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:32,423 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=474 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/B/5028a17b18c144d7a38e4721f6db56cf 2024-11-10T03:16:32,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/C/e8310040b8b1444fbc34bc140eed3750 is 50, key is test_row_0/C:col10/1731208590563/Put/seqid=0 2024-11-10T03:16:32,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741914_1090 (size=12301) 2024-11-10T03:16:32,493 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=474 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/C/e8310040b8b1444fbc34bc140eed3750 2024-11-10T03:16:32,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/A/cfe4475b44ee406e809238739dc232b0 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/cfe4475b44ee406e809238739dc232b0 2024-11-10T03:16:32,521 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/cfe4475b44ee406e809238739dc232b0, entries=150, sequenceid=474, filesize=12.0 K 2024-11-10T03:16:32,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/B/5028a17b18c144d7a38e4721f6db56cf as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/5028a17b18c144d7a38e4721f6db56cf 2024-11-10T03:16:32,544 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/5028a17b18c144d7a38e4721f6db56cf, entries=150, sequenceid=474, filesize=12.0 K 2024-11-10T03:16:32,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/C/e8310040b8b1444fbc34bc140eed3750 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/e8310040b8b1444fbc34bc140eed3750 2024-11-10T03:16:32,556 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/e8310040b8b1444fbc34bc140eed3750, entries=150, sequenceid=474, filesize=12.0 K 2024-11-10T03:16:32,557 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for 77ed36df592326523261438e65e5b722 in 990ms, sequenceid=474, compaction requested=false 2024-11-10T03:16:32,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 77ed36df592326523261438e65e5b722: 2024-11-10T03:16:32,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:32,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-10T03:16:32,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-10T03:16:32,562 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-10T03:16:32,562 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0720 sec 2024-11-10T03:16:32,565 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 2.0790 sec 2024-11-10T03:16:32,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-10T03:16:32,634 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T03:16:32,634 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T03:16:32,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-10T03:16:32,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-10T03:16:32,636 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T03:16:32,636 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T03:16:32,637 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T03:16:32,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-10T03:16:32,790 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-10T03:16:32,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:32,791 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 77ed36df592326523261438e65e5b722 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-10T03:16:32,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77ed36df592326523261438e65e5b722, store=A 2024-11-10T03:16:32,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:16:32,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77ed36df592326523261438e65e5b722, store=B 2024-11-10T03:16:32,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:16:32,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77ed36df592326523261438e65e5b722, store=C 2024-11-10T03:16:32,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:16:32,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/A/d792aafbb5dc4f1b8b5caef4a6d3a798 is 50, key is test_row_0/A:col10/1731208591728/Put/seqid=0 2024-11-10T03:16:32,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741915_1091 (size=12301) 2024-11-10T03:16:32,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on 77ed36df592326523261438e65e5b722 2024-11-10T03:16:32,898 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. as already flushing 2024-11-10T03:16:32,923 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:32,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51378 deadline: 1731208652919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:32,924 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:32,924 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:32,924 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:32,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51402 deadline: 1731208652921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:32,924 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:32,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:32,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:32,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:32,930 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:32,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51380 deadline: 1731208652923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:32,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:32,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51370 deadline: 1731208652923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:32,931 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:32,931 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:32,931 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:32,931 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:32,931 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:32,931 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:32,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-10T03:16:33,039 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:33,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51370 deadline: 1731208653035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:33,040 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:33,040 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:33,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51378 deadline: 1731208653035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:33,040 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:33,040 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:33,040 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:33,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51402 deadline: 1731208653035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:33,041 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:33,041 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:33,041 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:33,041 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:33,041 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:33,041 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:33,044 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:33,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51380 deadline: 1731208653043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:33,044 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:33,045 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:33,045 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:33,200 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=489 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/A/d792aafbb5dc4f1b8b5caef4a6d3a798 2024-11-10T03:16:33,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/B/fc5c4014748545bbad2d52631261bf32 is 50, key is test_row_0/B:col10/1731208591728/Put/seqid=0 2024-11-10T03:16:33,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741916_1092 (size=12301) 2024-11-10T03:16:33,260 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:33,260 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:33,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51378 deadline: 1731208653253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:33,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51370 deadline: 1731208653253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:33,260 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:33,260 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:33,260 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:33,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51402 deadline: 1731208653253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:33,260 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:33,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51380 deadline: 1731208653253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:33,260 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:33,261 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:33,261 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:33,261 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:33,261 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:33,261 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:33,261 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:33,261 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:33,261 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:33,262 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:33,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-10T03:16:33,565 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:33,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51370 deadline: 1731208653562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:33,566 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:33,566 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:33,566 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:33,573 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:33,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51378 deadline: 1731208653573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:33,574 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:33,574 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:33,574 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:33,574 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:33,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51402 deadline: 1731208653573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:33,574 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:33,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51380 deadline: 1731208653574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:33,576 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:33,576 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:33,576 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:33,576 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:33,576 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:33,576 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:33,616 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=489 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/B/fc5c4014748545bbad2d52631261bf32 2024-11-10T03:16:33,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/C/cb5bcfbbc6a846cda8034e740df56458 is 50, key is test_row_0/C:col10/1731208591728/Put/seqid=0 2024-11-10T03:16:33,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741917_1093 (size=12301) 2024-11-10T03:16:33,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-10T03:16:34,047 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=489 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/C/cb5bcfbbc6a846cda8034e740df56458 2024-11-10T03:16:34,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/A/d792aafbb5dc4f1b8b5caef4a6d3a798 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/d792aafbb5dc4f1b8b5caef4a6d3a798 2024-11-10T03:16:34,057 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/d792aafbb5dc4f1b8b5caef4a6d3a798, entries=150, sequenceid=489, filesize=12.0 K 2024-11-10T03:16:34,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/B/fc5c4014748545bbad2d52631261bf32 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/fc5c4014748545bbad2d52631261bf32 2024-11-10T03:16:34,068 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/fc5c4014748545bbad2d52631261bf32, entries=150, sequenceid=489, filesize=12.0 K 2024-11-10T03:16:34,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/C/cb5bcfbbc6a846cda8034e740df56458 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/cb5bcfbbc6a846cda8034e740df56458 2024-11-10T03:16:34,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:34,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51370 deadline: 1731208654074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:34,075 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:34,075 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:34,075 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:34,076 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/cb5bcfbbc6a846cda8034e740df56458, entries=150, sequenceid=489, filesize=12.0 K 2024-11-10T03:16:34,077 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 77ed36df592326523261438e65e5b722 in 1287ms, sequenceid=489, compaction requested=true 2024-11-10T03:16:34,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 77ed36df592326523261438e65e5b722: 2024-11-10T03:16:34,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:34,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-10T03:16:34,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-10T03:16:34,080 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-10T03:16:34,080 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4410 sec 2024-11-10T03:16:34,082 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 1.4470 sec 2024-11-10T03:16:34,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on 77ed36df592326523261438e65e5b722 2024-11-10T03:16:34,084 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 77ed36df592326523261438e65e5b722 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-10T03:16:34,084 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77ed36df592326523261438e65e5b722, store=A 2024-11-10T03:16:34,084 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:16:34,084 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77ed36df592326523261438e65e5b722, store=B 2024-11-10T03:16:34,085 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:16:34,085 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77ed36df592326523261438e65e5b722, store=C 2024-11-10T03:16:34,085 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:16:34,092 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/A/302b4243c3de4229be832eee7cb079de is 50, key is test_row_0/A:col10/1731208592920/Put/seqid=0 2024-11-10T03:16:34,104 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:34,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51380 deadline: 1731208654101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:34,104 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:34,104 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:34,104 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:34,108 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:34,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51402 deadline: 1731208654102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:34,109 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:34,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51378 deadline: 1731208654104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:34,109 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:34,109 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:34,109 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:34,109 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:34,109 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:34,109 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:34,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741918_1094 (size=14741) 2024-11-10T03:16:34,110 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=508 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/A/302b4243c3de4229be832eee7cb079de 2024-11-10T03:16:34,122 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/B/fdb6f80b87f046d999c3fcc04a0c7092 is 50, key is test_row_0/B:col10/1731208592920/Put/seqid=0 2024-11-10T03:16:34,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741919_1095 (size=12301) 2024-11-10T03:16:34,217 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:34,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51380 deadline: 1731208654213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:34,217 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:34,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51402 deadline: 1731208654213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:34,218 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:34,218 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:34,218 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:34,218 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:34,218 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:34,218 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:34,218 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:34,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51378 deadline: 1731208654213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:34,218 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:34,218 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:34,218 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:34,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:34,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51380 deadline: 1731208654423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:34,429 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:34,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:34,430 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:34,430 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:34,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51402 deadline: 1731208654423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:34,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51378 deadline: 1731208654423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:34,430 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:34,430 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:34,430 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:34,430 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:34,430 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:34,430 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:34,430 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:34,528 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=508 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/B/fdb6f80b87f046d999c3fcc04a0c7092 2024-11-10T03:16:34,537 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/C/7278817c59de40bcaddf47a2d0140b1e is 50, key is test_row_0/C:col10/1731208592920/Put/seqid=0 2024-11-10T03:16:34,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741920_1096 (size=12301) 2024-11-10T03:16:34,734 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:34,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51380 deadline: 1731208654732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:34,734 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:34,734 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:34,734 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:34,735 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:34,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51378 deadline: 1731208654732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:34,735 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:34,735 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:34,735 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:34,746 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:34,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51402 deadline: 1731208654743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:34,746 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:34,746 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:34,746 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:34,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-10T03:16:34,783 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T03:16:34,783 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T03:16:34,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-10T03:16:34,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-10T03:16:34,786 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T03:16:34,787 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T03:16:34,787 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T03:16:34,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-10T03:16:34,939 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-10T03:16:34,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:34,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. as already flushing 2024-11-10T03:16:34,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:34,940 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:34,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:34,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:34,947 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=508 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/C/7278817c59de40bcaddf47a2d0140b1e 2024-11-10T03:16:34,957 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/A/302b4243c3de4229be832eee7cb079de as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/302b4243c3de4229be832eee7cb079de 2024-11-10T03:16:34,964 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/302b4243c3de4229be832eee7cb079de, entries=200, sequenceid=508, filesize=14.4 K 2024-11-10T03:16:34,965 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/B/fdb6f80b87f046d999c3fcc04a0c7092 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/fdb6f80b87f046d999c3fcc04a0c7092 2024-11-10T03:16:34,974 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/fdb6f80b87f046d999c3fcc04a0c7092, entries=150, sequenceid=508, filesize=12.0 K 2024-11-10T03:16:34,975 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/C/7278817c59de40bcaddf47a2d0140b1e as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/7278817c59de40bcaddf47a2d0140b1e 2024-11-10T03:16:34,981 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/7278817c59de40bcaddf47a2d0140b1e, entries=150, sequenceid=508, filesize=12.0 K 2024-11-10T03:16:34,983 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 77ed36df592326523261438e65e5b722 in 898ms, sequenceid=508, compaction requested=true 2024-11-10T03:16:34,983 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 77ed36df592326523261438e65e5b722: 2024-11-10T03:16:34,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 77ed36df592326523261438e65e5b722:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T03:16:34,983 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T03:16:34,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:16:34,983 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T03:16:34,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 77ed36df592326523261438e65e5b722:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T03:16:34,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:16:34,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 77ed36df592326523261438e65e5b722:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T03:16:34,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T03:16:34,985 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52462 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T03:16:34,985 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50022 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T03:16:34,985 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1541): 77ed36df592326523261438e65e5b722/A is initiating minor compaction (all files) 2024-11-10T03:16:34,985 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1541): 77ed36df592326523261438e65e5b722/B is initiating minor compaction (all files) 2024-11-10T03:16:34,985 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 77ed36df592326523261438e65e5b722/A in TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:34,985 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 77ed36df592326523261438e65e5b722/B in TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:34,985 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/412be7d500184f6898983229b07f9784, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/cfe4475b44ee406e809238739dc232b0, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/d792aafbb5dc4f1b8b5caef4a6d3a798, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/302b4243c3de4229be832eee7cb079de] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp, totalSize=51.2 K 2024-11-10T03:16:34,985 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/9a7c66c8aec8411493926b38ef391bc3, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/5028a17b18c144d7a38e4721f6db56cf, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/fc5c4014748545bbad2d52631261bf32, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/fdb6f80b87f046d999c3fcc04a0c7092] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp, totalSize=48.8 K 2024-11-10T03:16:34,985 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a7c66c8aec8411493926b38ef391bc3, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=452, earliestPutTs=1731208590174 2024-11-10T03:16:34,986 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 412be7d500184f6898983229b07f9784, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=452, earliestPutTs=1731208590174 2024-11-10T03:16:34,986 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 5028a17b18c144d7a38e4721f6db56cf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=474, earliestPutTs=1731208590545 2024-11-10T03:16:34,987 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting cfe4475b44ee406e809238739dc232b0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=474, earliestPutTs=1731208590545 2024-11-10T03:16:34,987 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting fc5c4014748545bbad2d52631261bf32, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=489, earliestPutTs=1731208591728 2024-11-10T03:16:34,987 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting d792aafbb5dc4f1b8b5caef4a6d3a798, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=489, earliestPutTs=1731208591728 2024-11-10T03:16:34,987 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting fdb6f80b87f046d999c3fcc04a0c7092, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=508, earliestPutTs=1731208592920 2024-11-10T03:16:34,988 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 302b4243c3de4229be832eee7cb079de, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=508, earliestPutTs=1731208592917 2024-11-10T03:16:35,004 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 77ed36df592326523261438e65e5b722#B#compaction#84 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:16:35,005 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 77ed36df592326523261438e65e5b722#A#compaction#85 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:16:35,005 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/B/c9af7c0e745b4026a6c165f0d7d1c039 is 50, key is test_row_0/B:col10/1731208592920/Put/seqid=0 2024-11-10T03:16:35,005 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/A/9619a08b59e741ac8fcad5c01ef2c577 is 50, key is test_row_0/A:col10/1731208592920/Put/seqid=0 2024-11-10T03:16:35,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741921_1097 (size=13255) 2024-11-10T03:16:35,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741922_1098 (size=13255) 2024-11-10T03:16:35,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on 77ed36df592326523261438e65e5b722 2024-11-10T03:16:35,086 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 77ed36df592326523261438e65e5b722 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-10T03:16:35,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77ed36df592326523261438e65e5b722, store=A 2024-11-10T03:16:35,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:16:35,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77ed36df592326523261438e65e5b722, store=B 2024-11-10T03:16:35,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:16:35,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77ed36df592326523261438e65e5b722, store=C 2024-11-10T03:16:35,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:16:35,090 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/A/e9679b42b747452b8144bcbef2689d73 is 50, key is test_row_0/A:col10/1731208594099/Put/seqid=0 2024-11-10T03:16:35,095 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-10T03:16:35,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:35,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. as already flushing 2024-11-10T03:16:35,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:35,096 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:35,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:35,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:35,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-10T03:16:35,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741923_1099 (size=14741) 2024-11-10T03:16:35,109 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=522 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/A/e9679b42b747452b8144bcbef2689d73 2024-11-10T03:16:35,121 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/B/b0dc1941b0654086a68ba91c104a3d15 is 50, key is test_row_0/B:col10/1731208594099/Put/seqid=0 2024-11-10T03:16:35,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741924_1100 (size=12301) 2024-11-10T03:16:35,167 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:35,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51370 deadline: 1731208655165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:35,167 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:35,167 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:35,167 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:35,248 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:35,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51380 deadline: 1731208655242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:35,249 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-10T03:16:35,249 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:35,249 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:35,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:35,249 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:35,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51378 deadline: 1731208655242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:35,249 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:35,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. as already flushing 2024-11-10T03:16:35,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:35,249 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:35,249 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:35,249 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:35,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:35,249 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:35,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:35,254 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:35,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51402 deadline: 1731208655253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:35,254 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:35,254 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:35,254 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:35,279 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:35,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51370 deadline: 1731208655273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:35,280 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:35,280 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:35,280 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:35,401 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-10T03:16:35,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:35,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. as already flushing 2024-11-10T03:16:35,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:35,402 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:35,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:35,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:35,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-10T03:16:35,429 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/B/c9af7c0e745b4026a6c165f0d7d1c039 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/c9af7c0e745b4026a6c165f0d7d1c039 2024-11-10T03:16:35,437 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/A/9619a08b59e741ac8fcad5c01ef2c577 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/9619a08b59e741ac8fcad5c01ef2c577 2024-11-10T03:16:35,438 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 77ed36df592326523261438e65e5b722/B of 77ed36df592326523261438e65e5b722 into c9af7c0e745b4026a6c165f0d7d1c039(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:16:35,438 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 77ed36df592326523261438e65e5b722: 2024-11-10T03:16:35,438 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., storeName=77ed36df592326523261438e65e5b722/B, priority=12, startTime=1731208594983; duration=0sec 2024-11-10T03:16:35,438 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T03:16:35,438 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 77ed36df592326523261438e65e5b722:B 2024-11-10T03:16:35,439 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T03:16:35,442 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50022 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T03:16:35,442 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1541): 77ed36df592326523261438e65e5b722/C is initiating minor compaction (all files) 2024-11-10T03:16:35,442 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 77ed36df592326523261438e65e5b722/C in TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:35,442 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/49c70c22bf1e4a15a27d5c8c7b7137ee, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/e8310040b8b1444fbc34bc140eed3750, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/cb5bcfbbc6a846cda8034e740df56458, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/7278817c59de40bcaddf47a2d0140b1e] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp, totalSize=48.8 K 2024-11-10T03:16:35,444 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 49c70c22bf1e4a15a27d5c8c7b7137ee, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=452, earliestPutTs=1731208590174 2024-11-10T03:16:35,444 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 77ed36df592326523261438e65e5b722/A of 77ed36df592326523261438e65e5b722 into 9619a08b59e741ac8fcad5c01ef2c577(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:16:35,444 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 77ed36df592326523261438e65e5b722: 2024-11-10T03:16:35,444 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., storeName=77ed36df592326523261438e65e5b722/A, priority=12, startTime=1731208594983; duration=0sec 2024-11-10T03:16:35,444 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting e8310040b8b1444fbc34bc140eed3750, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=474, earliestPutTs=1731208590545 2024-11-10T03:16:35,445 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting cb5bcfbbc6a846cda8034e740df56458, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=489, earliestPutTs=1731208591728 2024-11-10T03:16:35,445 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 7278817c59de40bcaddf47a2d0140b1e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=508, earliestPutTs=1731208592920 2024-11-10T03:16:35,449 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:16:35,449 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 77ed36df592326523261438e65e5b722:A 2024-11-10T03:16:35,458 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 77ed36df592326523261438e65e5b722#C#compaction#88 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:16:35,459 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/C/05f182cc6b2e46ecbde195e9994c0226 is 50, key is test_row_0/C:col10/1731208592920/Put/seqid=0 2024-11-10T03:16:35,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:35,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51370 deadline: 1731208655484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:35,492 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:35,493 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:35,493 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:35,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741925_1101 (size=13255) 2024-11-10T03:16:35,521 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/C/05f182cc6b2e46ecbde195e9994c0226 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/05f182cc6b2e46ecbde195e9994c0226 2024-11-10T03:16:35,539 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=522 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/B/b0dc1941b0654086a68ba91c104a3d15 2024-11-10T03:16:35,541 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 77ed36df592326523261438e65e5b722/C of 77ed36df592326523261438e65e5b722 into 05f182cc6b2e46ecbde195e9994c0226(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:16:35,541 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 77ed36df592326523261438e65e5b722: 2024-11-10T03:16:35,541 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., storeName=77ed36df592326523261438e65e5b722/C, priority=12, startTime=1731208594983; duration=0sec 2024-11-10T03:16:35,541 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:16:35,541 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 77ed36df592326523261438e65e5b722:C 2024-11-10T03:16:35,555 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-10T03:16:35,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:35,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. as already flushing 2024-11-10T03:16:35,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:35,555 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:35,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:35,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:35,585 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/C/2e5acc6f0cc4442282aa61d9f70ce988 is 50, key is test_row_0/C:col10/1731208594099/Put/seqid=0 2024-11-10T03:16:35,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741926_1102 (size=12301) 2024-11-10T03:16:35,712 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-10T03:16:35,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:35,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. as already flushing 2024-11-10T03:16:35,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:35,713 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:35,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:35,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:35,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:35,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51370 deadline: 1731208655803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:35,804 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:35,804 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:35,804 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:35,866 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-10T03:16:35,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:35,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. as already flushing 2024-11-10T03:16:35,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:35,866 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:35,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:35,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:35,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-10T03:16:36,008 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=522 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/C/2e5acc6f0cc4442282aa61d9f70ce988 2024-11-10T03:16:36,017 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/A/e9679b42b747452b8144bcbef2689d73 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/e9679b42b747452b8144bcbef2689d73 2024-11-10T03:16:36,020 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-10T03:16:36,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:36,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. as already flushing 2024-11-10T03:16:36,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:36,020 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:36,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:36,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:36,025 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/e9679b42b747452b8144bcbef2689d73, entries=200, sequenceid=522, filesize=14.4 K 2024-11-10T03:16:36,026 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/B/b0dc1941b0654086a68ba91c104a3d15 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/b0dc1941b0654086a68ba91c104a3d15 2024-11-10T03:16:36,031 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/b0dc1941b0654086a68ba91c104a3d15, entries=150, sequenceid=522, filesize=12.0 K 2024-11-10T03:16:36,032 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/C/2e5acc6f0cc4442282aa61d9f70ce988 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/2e5acc6f0cc4442282aa61d9f70ce988 2024-11-10T03:16:36,036 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/2e5acc6f0cc4442282aa61d9f70ce988, entries=150, sequenceid=522, filesize=12.0 K 2024-11-10T03:16:36,037 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 77ed36df592326523261438e65e5b722 in 952ms, sequenceid=522, compaction requested=false 2024-11-10T03:16:36,037 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 77ed36df592326523261438e65e5b722: 2024-11-10T03:16:36,174 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-10T03:16:36,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:36,174 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 77ed36df592326523261438e65e5b722 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-10T03:16:36,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77ed36df592326523261438e65e5b722, store=A 2024-11-10T03:16:36,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:16:36,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77ed36df592326523261438e65e5b722, store=B 2024-11-10T03:16:36,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:16:36,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77ed36df592326523261438e65e5b722, store=C 2024-11-10T03:16:36,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:16:36,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/A/ac127a4170c849948ea0148ae7facb3a is 50, key is test_row_0/A:col10/1731208595164/Put/seqid=0 2024-11-10T03:16:36,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741927_1103 (size=12301) 2024-11-10T03:16:36,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on 77ed36df592326523261438e65e5b722 2024-11-10T03:16:36,255 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. as already flushing 2024-11-10T03:16:36,282 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:36,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51378 deadline: 1731208656275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:36,283 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:36,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51380 deadline: 1731208656277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:36,283 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:36,283 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:36,283 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:36,283 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:36,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51402 deadline: 1731208656281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:36,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:36,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:36,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:36,287 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:36,287 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:36,287 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:36,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:36,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51370 deadline: 1731208656314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:36,316 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:36,316 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:36,316 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:36,396 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:36,396 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:36,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51402 deadline: 1731208656393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:36,396 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:36,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51380 deadline: 1731208656394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:36,399 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:36,399 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:36,399 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:36,399 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:36,399 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:36,399 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:36,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51378 deadline: 1731208656393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:36,402 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:36,402 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:36,402 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:36,586 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=543 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/A/ac127a4170c849948ea0148ae7facb3a 2024-11-10T03:16:36,613 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:36,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51402 deadline: 1731208656603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:36,614 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:36,614 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:36,614 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:36,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/B/56e26afc7ed54d2382b3210a9f7c54e2 is 50, key is test_row_0/B:col10/1731208595164/Put/seqid=0 2024-11-10T03:16:36,614 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:36,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51380 deadline: 1731208656608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:36,615 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:36,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51378 deadline: 1731208656613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:36,615 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:36,616 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:36,616 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:36,618 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:36,618 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:36,618 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:36,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741928_1104 (size=12301) 2024-11-10T03:16:36,670 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-10T03:16:36,670 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-10T03:16:36,670 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T03:16:36,670 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T03:16:36,675 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-10T03:16:36,675 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T03:16:36,675 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:16:36,675 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:16:36,675 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:16:36,676 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:16:36,676 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T03:16:36,677 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:16:36,677 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:16:36,677 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T03:16:36,680 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T03:16:36,682 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-10T03:16:36,682 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T03:16:36,682 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:16:36,682 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:16:36,682 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T03:16:36,924 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:36,924 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:36,924 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:36,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51402 deadline: 1731208656923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:36,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51378 deadline: 1731208656924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:36,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51380 deadline: 1731208656923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:36,925 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:36,925 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:36,925 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:36,925 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:36,925 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:36,925 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:36,925 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:36,925 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:36,925 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:16:36,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-10T03:16:37,032 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=543 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/B/56e26afc7ed54d2382b3210a9f7c54e2 2024-11-10T03:16:37,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/C/6f9c637a891f492fb081b088e0fd61ca is 50, key is test_row_0/C:col10/1731208595164/Put/seqid=0 2024-11-10T03:16:37,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741929_1105 (size=12301) 2024-11-10T03:16:37,053 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=543 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/C/6f9c637a891f492fb081b088e0fd61ca 2024-11-10T03:16:37,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/A/ac127a4170c849948ea0148ae7facb3a as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/ac127a4170c849948ea0148ae7facb3a 2024-11-10T03:16:37,075 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/ac127a4170c849948ea0148ae7facb3a, entries=150, sequenceid=543, filesize=12.0 K 2024-11-10T03:16:37,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/B/56e26afc7ed54d2382b3210a9f7c54e2 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/56e26afc7ed54d2382b3210a9f7c54e2 2024-11-10T03:16:37,082 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/56e26afc7ed54d2382b3210a9f7c54e2, entries=150, sequenceid=543, filesize=12.0 K 2024-11-10T03:16:37,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/C/6f9c637a891f492fb081b088e0fd61ca as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/6f9c637a891f492fb081b088e0fd61ca 2024-11-10T03:16:37,092 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/6f9c637a891f492fb081b088e0fd61ca, entries=150, sequenceid=543, filesize=12.0 K 2024-11-10T03:16:37,093 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 77ed36df592326523261438e65e5b722 in 919ms, sequenceid=543, compaction requested=true 2024-11-10T03:16:37,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 77ed36df592326523261438e65e5b722: 2024-11-10T03:16:37,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:37,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-10T03:16:37,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-10T03:16:37,098 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-10T03:16:37,098 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3090 sec 2024-11-10T03:16:37,101 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 2.3160 sec 2024-11-10T03:16:37,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on 77ed36df592326523261438e65e5b722 2024-11-10T03:16:37,335 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 77ed36df592326523261438e65e5b722 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-10T03:16:37,336 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-10T03:16:37,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77ed36df592326523261438e65e5b722, store=A 2024-11-10T03:16:37,336 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T03:16:37,336 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:16:37,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:16:37,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77ed36df592326523261438e65e5b722, store=B 2024-11-10T03:16:37,336 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:16:37,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:16:37,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77ed36df592326523261438e65e5b722, store=C 2024-11-10T03:16:37,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:16:37,336 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T03:16:37,341 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/A/a22ca9a06312479895d2870872263194 is 50, key is test_row_0/A:col10/1731208596268/Put/seqid=0 2024-11-10T03:16:37,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741930_1106 (size=12301) 2024-11-10T03:16:37,389 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-10T03:16:37,390 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T03:16:37,390 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:16:37,390 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:16:37,390 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T03:16:37,441 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-10T03:16:37,441 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-10T03:16:37,441 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T03:16:37,441 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T03:16:37,441 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:16:37,441 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:16:37,441 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:16:37,441 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:16:37,441 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T03:16:37,441 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-10T03:16:37,441 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T03:16:37,442 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:16:37,442 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:16:37,442 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T03:16:37,444 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T03:16:37,753 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=558 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/A/a22ca9a06312479895d2870872263194 2024-11-10T03:16:37,764 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/B/6c8489ed907444c2bb9ab7083aa9356d is 50, key is test_row_0/B:col10/1731208596268/Put/seqid=0 2024-11-10T03:16:37,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741931_1107 (size=12301) 2024-11-10T03:16:37,771 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=558 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/B/6c8489ed907444c2bb9ab7083aa9356d 2024-11-10T03:16:37,790 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/C/ad0383fa138f4d0f9ff8684dd9568a48 is 50, key is test_row_0/C:col10/1731208596268/Put/seqid=0 2024-11-10T03:16:37,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741932_1108 (size=12301) 2024-11-10T03:16:38,195 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=558 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/C/ad0383fa138f4d0f9ff8684dd9568a48 2024-11-10T03:16:38,203 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/A/a22ca9a06312479895d2870872263194 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/a22ca9a06312479895d2870872263194 2024-11-10T03:16:38,208 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/a22ca9a06312479895d2870872263194, entries=150, sequenceid=558, filesize=12.0 K 2024-11-10T03:16:38,209 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/B/6c8489ed907444c2bb9ab7083aa9356d as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/6c8489ed907444c2bb9ab7083aa9356d 2024-11-10T03:16:38,217 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/6c8489ed907444c2bb9ab7083aa9356d, entries=150, sequenceid=558, filesize=12.0 K 2024-11-10T03:16:38,221 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/C/ad0383fa138f4d0f9ff8684dd9568a48 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/ad0383fa138f4d0f9ff8684dd9568a48 2024-11-10T03:16:38,233 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/ad0383fa138f4d0f9ff8684dd9568a48, entries=150, sequenceid=558, filesize=12.0 K 2024-11-10T03:16:38,234 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=26.84 KB/27480 for 77ed36df592326523261438e65e5b722 in 899ms, sequenceid=558, compaction requested=true 2024-11-10T03:16:38,234 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 77ed36df592326523261438e65e5b722: 2024-11-10T03:16:38,234 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T03:16:38,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 77ed36df592326523261438e65e5b722:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T03:16:38,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:16:38,235 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T03:16:38,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 77ed36df592326523261438e65e5b722:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T03:16:38,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:16:38,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 77ed36df592326523261438e65e5b722:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T03:16:38,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T03:16:38,237 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50158 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T03:16:38,237 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1541): 77ed36df592326523261438e65e5b722/B is initiating minor compaction (all files) 2024-11-10T03:16:38,237 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 77ed36df592326523261438e65e5b722/B in TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:38,238 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/c9af7c0e745b4026a6c165f0d7d1c039, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/b0dc1941b0654086a68ba91c104a3d15, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/56e26afc7ed54d2382b3210a9f7c54e2, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/6c8489ed907444c2bb9ab7083aa9356d] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp, totalSize=49.0 K 2024-11-10T03:16:38,239 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting c9af7c0e745b4026a6c165f0d7d1c039, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=508, earliestPutTs=1731208592920 2024-11-10T03:16:38,239 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting b0dc1941b0654086a68ba91c104a3d15, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=522, earliestPutTs=1731208594098 2024-11-10T03:16:38,240 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 56e26afc7ed54d2382b3210a9f7c54e2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=543, earliestPutTs=1731208595152 2024-11-10T03:16:38,241 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52598 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T03:16:38,242 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1541): 77ed36df592326523261438e65e5b722/A is initiating minor compaction (all files) 2024-11-10T03:16:38,242 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 77ed36df592326523261438e65e5b722/A in TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:38,242 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/9619a08b59e741ac8fcad5c01ef2c577, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/e9679b42b747452b8144bcbef2689d73, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/ac127a4170c849948ea0148ae7facb3a, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/a22ca9a06312479895d2870872263194] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp, totalSize=51.4 K 2024-11-10T03:16:38,242 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c8489ed907444c2bb9ab7083aa9356d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=558, earliestPutTs=1731208596268 2024-11-10T03:16:38,243 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9619a08b59e741ac8fcad5c01ef2c577, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=508, earliestPutTs=1731208592920 2024-11-10T03:16:38,244 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting e9679b42b747452b8144bcbef2689d73, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=522, earliestPutTs=1731208594098 2024-11-10T03:16:38,245 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting ac127a4170c849948ea0148ae7facb3a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=543, earliestPutTs=1731208595152 2024-11-10T03:16:38,246 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting a22ca9a06312479895d2870872263194, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=558, earliestPutTs=1731208596268 2024-11-10T03:16:38,262 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 77ed36df592326523261438e65e5b722#B#compaction#96 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:16:38,263 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/B/a21065c06a3945fe94263440f718980a is 50, key is test_row_0/B:col10/1731208596268/Put/seqid=0 2024-11-10T03:16:38,268 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 77ed36df592326523261438e65e5b722#A#compaction#97 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:16:38,268 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/A/89ffb4f7e0b64e918b8fb0f761983762 is 50, key is test_row_0/A:col10/1731208596268/Put/seqid=0 2024-11-10T03:16:38,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741933_1109 (size=13391) 2024-11-10T03:16:38,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741934_1110 (size=13391) 2024-11-10T03:16:38,336 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/A/89ffb4f7e0b64e918b8fb0f761983762 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/89ffb4f7e0b64e918b8fb0f761983762 2024-11-10T03:16:38,344 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 77ed36df592326523261438e65e5b722/A of 77ed36df592326523261438e65e5b722 into 89ffb4f7e0b64e918b8fb0f761983762(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:16:38,345 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 77ed36df592326523261438e65e5b722: 2024-11-10T03:16:38,345 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., storeName=77ed36df592326523261438e65e5b722/A, priority=12, startTime=1731208598234; duration=0sec 2024-11-10T03:16:38,345 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T03:16:38,345 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 77ed36df592326523261438e65e5b722:A 2024-11-10T03:16:38,345 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T03:16:38,347 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50158 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T03:16:38,347 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1541): 77ed36df592326523261438e65e5b722/C is initiating minor compaction (all files) 2024-11-10T03:16:38,347 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 77ed36df592326523261438e65e5b722/C in TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:38,347 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/05f182cc6b2e46ecbde195e9994c0226, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/2e5acc6f0cc4442282aa61d9f70ce988, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/6f9c637a891f492fb081b088e0fd61ca, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/ad0383fa138f4d0f9ff8684dd9568a48] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp, totalSize=49.0 K 2024-11-10T03:16:38,348 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 05f182cc6b2e46ecbde195e9994c0226, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=508, earliestPutTs=1731208592920 2024-11-10T03:16:38,348 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2e5acc6f0cc4442282aa61d9f70ce988, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=522, earliestPutTs=1731208594098 2024-11-10T03:16:38,351 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6f9c637a891f492fb081b088e0fd61ca, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=543, earliestPutTs=1731208595152 2024-11-10T03:16:38,352 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad0383fa138f4d0f9ff8684dd9568a48, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=558, earliestPutTs=1731208596268 2024-11-10T03:16:38,365 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 77ed36df592326523261438e65e5b722#C#compaction#98 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:16:38,366 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/C/cb0a9f36e98c487b985bacec3d346df0 is 50, key is test_row_0/C:col10/1731208596268/Put/seqid=0 2024-11-10T03:16:38,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741935_1111 (size=13391) 2024-11-10T03:16:38,719 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/B/a21065c06a3945fe94263440f718980a as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/a21065c06a3945fe94263440f718980a 2024-11-10T03:16:38,727 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 77ed36df592326523261438e65e5b722/B of 77ed36df592326523261438e65e5b722 into a21065c06a3945fe94263440f718980a(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:16:38,727 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 77ed36df592326523261438e65e5b722: 2024-11-10T03:16:38,727 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., storeName=77ed36df592326523261438e65e5b722/B, priority=12, startTime=1731208598235; duration=0sec 2024-11-10T03:16:38,727 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:16:38,727 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 77ed36df592326523261438e65e5b722:B 2024-11-10T03:16:38,778 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/C/cb0a9f36e98c487b985bacec3d346df0 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/cb0a9f36e98c487b985bacec3d346df0 2024-11-10T03:16:38,784 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 77ed36df592326523261438e65e5b722/C of 77ed36df592326523261438e65e5b722 into cb0a9f36e98c487b985bacec3d346df0(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:16:38,784 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 77ed36df592326523261438e65e5b722: 2024-11-10T03:16:38,784 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., storeName=77ed36df592326523261438e65e5b722/C, priority=12, startTime=1731208598235; duration=0sec 2024-11-10T03:16:38,784 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:16:38,784 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 77ed36df592326523261438e65e5b722:C 2024-11-10T03:16:38,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-10T03:16:38,944 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T03:16:38,944 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-10T03:16:38,944 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 120 2024-11-10T03:16:38,944 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 127 2024-11-10T03:16:38,944 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 123 2024-11-10T03:16:38,944 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 76 2024-11-10T03:16:38,944 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 19 2024-11-10T03:16:38,944 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-10T03:16:38,944 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2152 2024-11-10T03:16:38,944 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2193 2024-11-10T03:16:38,944 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-10T03:16:38,944 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 994 2024-11-10T03:16:38,944 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2980 rows 2024-11-10T03:16:38,944 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1000 2024-11-10T03:16:38,944 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2995 rows 2024-11-10T03:16:38,944 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T03:16:38,944 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T03:16:38,944 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:16:38,945 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:16:38,945 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T03:16:38,948 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-10T03:16:38,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-10T03:16:38,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-10T03:16:38,955 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731208598954"}]},"ts":"1731208598954"} 2024-11-10T03:16:38,957 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-10T03:16:38,957 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-10T03:16:38,959 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-10T03:16:38,963 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=77ed36df592326523261438e65e5b722, UNASSIGN}] 2024-11-10T03:16:38,964 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=77ed36df592326523261438e65e5b722, UNASSIGN 2024-11-10T03:16:38,965 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=77ed36df592326523261438e65e5b722, regionState=CLOSING, regionLocation=394d2222f724,38139,1731208572054 2024-11-10T03:16:38,968 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=77ed36df592326523261438e65e5b722, UNASSIGN because future has completed 2024-11-10T03:16:38,968 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-10T03:16:38,969 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; CloseRegionProcedure 77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054}] 2024-11-10T03:16:39,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-10T03:16:39,126 INFO [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] handler.UnassignRegionHandler(122): Close 77ed36df592326523261438e65e5b722 2024-11-10T03:16:39,126 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-10T03:16:39,127 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1722): Closing 77ed36df592326523261438e65e5b722, disabling compactions & flushes 2024-11-10T03:16:39,127 INFO [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:39,127 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:39,127 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. after waiting 0 ms 2024-11-10T03:16:39,127 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:39,128 INFO [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(2902): Flushing 77ed36df592326523261438e65e5b722 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-10T03:16:39,128 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77ed36df592326523261438e65e5b722, store=A 2024-11-10T03:16:39,128 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:16:39,128 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77ed36df592326523261438e65e5b722, store=B 2024-11-10T03:16:39,128 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:16:39,128 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77ed36df592326523261438e65e5b722, store=C 2024-11-10T03:16:39,128 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:16:39,194 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/A/cbb970cf5767461f85160b19057c95fc is 50, key is test_row_0/A:col10/1731208597434/Put/seqid=0 2024-11-10T03:16:39,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741936_1112 (size=9857) 2024-11-10T03:16:39,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-10T03:16:39,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-10T03:16:39,604 INFO [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=568 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/A/cbb970cf5767461f85160b19057c95fc 2024-11-10T03:16:39,613 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/B/7f5e84ac091e4713b3f6c790525b3f0c is 50, key is test_row_0/B:col10/1731208597434/Put/seqid=0 2024-11-10T03:16:39,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741937_1113 (size=9857) 2024-11-10T03:16:40,019 INFO [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=568 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/B/7f5e84ac091e4713b3f6c790525b3f0c 2024-11-10T03:16:40,026 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/C/046b7e6bf0a14e8e952f9b5d8542e89d is 50, key is test_row_0/C:col10/1731208597434/Put/seqid=0 2024-11-10T03:16:40,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741938_1114 (size=9857) 2024-11-10T03:16:40,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-10T03:16:40,311 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-10T03:16:40,430 INFO [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=568 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/C/046b7e6bf0a14e8e952f9b5d8542e89d 2024-11-10T03:16:40,436 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/A/cbb970cf5767461f85160b19057c95fc as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/cbb970cf5767461f85160b19057c95fc 2024-11-10T03:16:40,443 INFO [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/cbb970cf5767461f85160b19057c95fc, entries=100, sequenceid=568, filesize=9.6 K 2024-11-10T03:16:40,444 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/B/7f5e84ac091e4713b3f6c790525b3f0c as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/7f5e84ac091e4713b3f6c790525b3f0c 2024-11-10T03:16:40,459 INFO [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/7f5e84ac091e4713b3f6c790525b3f0c, entries=100, sequenceid=568, filesize=9.6 K 2024-11-10T03:16:40,460 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/.tmp/C/046b7e6bf0a14e8e952f9b5d8542e89d as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/046b7e6bf0a14e8e952f9b5d8542e89d 2024-11-10T03:16:40,466 INFO [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/046b7e6bf0a14e8e952f9b5d8542e89d, entries=100, sequenceid=568, filesize=9.6 K 2024-11-10T03:16:40,467 INFO [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 77ed36df592326523261438e65e5b722 in 1340ms, sequenceid=568, compaction requested=false 2024-11-10T03:16:40,467 DEBUG [StoreCloser-TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/67b942361abf4722b46bb67189b79d92, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/c5f104db49ae4b59ba400cb62edc6433, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/f4be510c6299423b9337909a8a491132, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/d56b8bbae5724177ba16577be40d3aea, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/64d58ffdf77b4ba9b1d97305aff60dc9, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/23440f3e46be452a9cb836b5053d0410, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/78728cc51c274beca3469b062eb28e4e, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/5ce8d689f149415e9a6ec3b82e7705aa, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/05da6396202d45bf88a07b86dd9a80fc, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/655e7a1e8dd24177a4bf8dc70955149c, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/60e12467571048228a11f35b56626d52, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/a1ddef07d63b4a548613f14785b28374, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/dd96679e1c87434796be5697903956e5, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/e9f5ad945db14b4c89effed4375ba61b, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/62f0a998e40549b8911742c7d484727b, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/905a50fe0e4945c580f1203d43d41c3f, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/523e5f6c57814d5490dbe2d0e58cf575, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/f0846beea3b84d4891e5da0afd8439f7, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/4d15eb855984493994e110fdda9a2954, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/b2f5afa9366949f8bd146386f28b2ece, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/558ad750b42646beade61cee98057759, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/d263d8ba23164bee8d31995ddae0034e, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/c71a699f5dc642ddaaff60c8e1623c27, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/5e6420f527874f4a9ec60d977680e4c8, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/412be7d500184f6898983229b07f9784, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/cfe4475b44ee406e809238739dc232b0, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/d792aafbb5dc4f1b8b5caef4a6d3a798, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/302b4243c3de4229be832eee7cb079de, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/9619a08b59e741ac8fcad5c01ef2c577, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/e9679b42b747452b8144bcbef2689d73, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/ac127a4170c849948ea0148ae7facb3a, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/a22ca9a06312479895d2870872263194] to archive 2024-11-10T03:16:40,479 DEBUG [StoreCloser-TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T03:16:40,483 DEBUG [StoreCloser-TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/67b942361abf4722b46bb67189b79d92 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/67b942361abf4722b46bb67189b79d92 2024-11-10T03:16:40,485 DEBUG [StoreCloser-TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/c5f104db49ae4b59ba400cb62edc6433 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/c5f104db49ae4b59ba400cb62edc6433 2024-11-10T03:16:40,486 DEBUG [StoreCloser-TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/f4be510c6299423b9337909a8a491132 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/f4be510c6299423b9337909a8a491132 2024-11-10T03:16:40,488 DEBUG [StoreCloser-TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/d56b8bbae5724177ba16577be40d3aea to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/d56b8bbae5724177ba16577be40d3aea 2024-11-10T03:16:40,489 DEBUG [StoreCloser-TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/64d58ffdf77b4ba9b1d97305aff60dc9 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/64d58ffdf77b4ba9b1d97305aff60dc9 2024-11-10T03:16:40,490 DEBUG [StoreCloser-TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/23440f3e46be452a9cb836b5053d0410 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/23440f3e46be452a9cb836b5053d0410 2024-11-10T03:16:40,492 DEBUG [StoreCloser-TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/78728cc51c274beca3469b062eb28e4e to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/78728cc51c274beca3469b062eb28e4e 2024-11-10T03:16:40,493 DEBUG [StoreCloser-TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/5ce8d689f149415e9a6ec3b82e7705aa to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/5ce8d689f149415e9a6ec3b82e7705aa 2024-11-10T03:16:40,494 DEBUG [StoreCloser-TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/05da6396202d45bf88a07b86dd9a80fc to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/05da6396202d45bf88a07b86dd9a80fc 2024-11-10T03:16:40,495 DEBUG [StoreCloser-TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/655e7a1e8dd24177a4bf8dc70955149c to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/655e7a1e8dd24177a4bf8dc70955149c 2024-11-10T03:16:40,497 DEBUG [StoreCloser-TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/60e12467571048228a11f35b56626d52 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/60e12467571048228a11f35b56626d52 2024-11-10T03:16:40,498 DEBUG [StoreCloser-TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/a1ddef07d63b4a548613f14785b28374 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/a1ddef07d63b4a548613f14785b28374 2024-11-10T03:16:40,499 DEBUG [StoreCloser-TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/dd96679e1c87434796be5697903956e5 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/dd96679e1c87434796be5697903956e5 2024-11-10T03:16:40,500 DEBUG [StoreCloser-TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/e9f5ad945db14b4c89effed4375ba61b to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/e9f5ad945db14b4c89effed4375ba61b 2024-11-10T03:16:40,501 DEBUG [StoreCloser-TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/62f0a998e40549b8911742c7d484727b to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/62f0a998e40549b8911742c7d484727b 2024-11-10T03:16:40,502 DEBUG [StoreCloser-TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/905a50fe0e4945c580f1203d43d41c3f to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/905a50fe0e4945c580f1203d43d41c3f 2024-11-10T03:16:40,503 DEBUG [StoreCloser-TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/523e5f6c57814d5490dbe2d0e58cf575 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/523e5f6c57814d5490dbe2d0e58cf575 2024-11-10T03:16:40,504 DEBUG [StoreCloser-TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/f0846beea3b84d4891e5da0afd8439f7 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/f0846beea3b84d4891e5da0afd8439f7 2024-11-10T03:16:40,505 DEBUG [StoreCloser-TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/4d15eb855984493994e110fdda9a2954 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/4d15eb855984493994e110fdda9a2954 2024-11-10T03:16:40,506 DEBUG [StoreCloser-TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/b2f5afa9366949f8bd146386f28b2ece to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/b2f5afa9366949f8bd146386f28b2ece 2024-11-10T03:16:40,507 DEBUG [StoreCloser-TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/558ad750b42646beade61cee98057759 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/558ad750b42646beade61cee98057759 2024-11-10T03:16:40,508 DEBUG [StoreCloser-TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/d263d8ba23164bee8d31995ddae0034e to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/d263d8ba23164bee8d31995ddae0034e 2024-11-10T03:16:40,510 DEBUG [StoreCloser-TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/c71a699f5dc642ddaaff60c8e1623c27 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/c71a699f5dc642ddaaff60c8e1623c27 2024-11-10T03:16:40,511 DEBUG [StoreCloser-TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/5e6420f527874f4a9ec60d977680e4c8 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/5e6420f527874f4a9ec60d977680e4c8 2024-11-10T03:16:40,512 DEBUG [StoreCloser-TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/412be7d500184f6898983229b07f9784 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/412be7d500184f6898983229b07f9784 2024-11-10T03:16:40,513 DEBUG [StoreCloser-TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/cfe4475b44ee406e809238739dc232b0 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/cfe4475b44ee406e809238739dc232b0 2024-11-10T03:16:40,514 DEBUG [StoreCloser-TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/d792aafbb5dc4f1b8b5caef4a6d3a798 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/d792aafbb5dc4f1b8b5caef4a6d3a798 2024-11-10T03:16:40,515 DEBUG [StoreCloser-TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/302b4243c3de4229be832eee7cb079de to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/302b4243c3de4229be832eee7cb079de 2024-11-10T03:16:40,516 DEBUG [StoreCloser-TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/9619a08b59e741ac8fcad5c01ef2c577 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/9619a08b59e741ac8fcad5c01ef2c577 2024-11-10T03:16:40,518 DEBUG [StoreCloser-TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/e9679b42b747452b8144bcbef2689d73 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/e9679b42b747452b8144bcbef2689d73 2024-11-10T03:16:40,519 DEBUG [StoreCloser-TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/ac127a4170c849948ea0148ae7facb3a to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/ac127a4170c849948ea0148ae7facb3a 2024-11-10T03:16:40,520 DEBUG [StoreCloser-TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/a22ca9a06312479895d2870872263194 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/a22ca9a06312479895d2870872263194 2024-11-10T03:16:40,536 DEBUG [StoreCloser-TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/1df2dd6e105343ea8b21dffafc8b9927, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/a50baa4568a24aceb7aafe83deab7ae3, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/460d9c2f418041a59e01afada5998cec, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/ecf1fba204b1447aa8318898c5576ac1, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/7645396c537e44168393ed1b88f13433, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/8a6802dd4c034e5697d930ab594c0dfc, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/9524576126ef48498ed311aac0243f91, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/638046dd97a74b329485a9c6cfe40515, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/9627444047fa4fe2865f20f167afcf0d, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/c70b7d9bc2614bb985329243a5c9ae0f, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/acfca18bf378472cadd8637388d72a74, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/852f97728a4e49d18f38f2da39fd4ba2, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/239578cf7394453b807f66cc0d7133cb, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/d12a9c8d0b4042ec9d3dc069b11b56a8, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/bddf2f14f2d24a3ea6fa9994e7e6199a, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/e4c7aceabb9d4b6d9ec6ece931eba970, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/71f31734b5b14fe7ba1b946619380fad, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/ed4f506be2e849638e5e2c1501cd0d40, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/5341143ae1c3426aa28c71e4527585c3, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/50195dcec2904f42833edd14a5f180ad, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/4c1ecc6e66a3480391ba74e668ee873d, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/38a7848ea2a64fe8951b74af9b96f014, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/5f62f3d4e99e4491a40b85e0ba9a7f37, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/9a7c66c8aec8411493926b38ef391bc3, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/85a2ab167dbf4767bc1bc36fd43ea015, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/5028a17b18c144d7a38e4721f6db56cf, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/fc5c4014748545bbad2d52631261bf32, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/c9af7c0e745b4026a6c165f0d7d1c039, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/fdb6f80b87f046d999c3fcc04a0c7092, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/b0dc1941b0654086a68ba91c104a3d15, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/56e26afc7ed54d2382b3210a9f7c54e2, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/6c8489ed907444c2bb9ab7083aa9356d] to archive 2024-11-10T03:16:40,537 DEBUG [StoreCloser-TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T03:16:40,539 DEBUG [StoreCloser-TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/1df2dd6e105343ea8b21dffafc8b9927 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/1df2dd6e105343ea8b21dffafc8b9927 2024-11-10T03:16:40,540 DEBUG [StoreCloser-TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/a50baa4568a24aceb7aafe83deab7ae3 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/a50baa4568a24aceb7aafe83deab7ae3 2024-11-10T03:16:40,542 DEBUG [StoreCloser-TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/460d9c2f418041a59e01afada5998cec to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/460d9c2f418041a59e01afada5998cec 2024-11-10T03:16:40,543 DEBUG [StoreCloser-TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/ecf1fba204b1447aa8318898c5576ac1 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/ecf1fba204b1447aa8318898c5576ac1 2024-11-10T03:16:40,545 DEBUG [StoreCloser-TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/7645396c537e44168393ed1b88f13433 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/7645396c537e44168393ed1b88f13433 2024-11-10T03:16:40,546 DEBUG [StoreCloser-TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/8a6802dd4c034e5697d930ab594c0dfc to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/8a6802dd4c034e5697d930ab594c0dfc 2024-11-10T03:16:40,548 DEBUG [StoreCloser-TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/9524576126ef48498ed311aac0243f91 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/9524576126ef48498ed311aac0243f91 2024-11-10T03:16:40,549 DEBUG [StoreCloser-TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/638046dd97a74b329485a9c6cfe40515 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/638046dd97a74b329485a9c6cfe40515 2024-11-10T03:16:40,550 DEBUG [StoreCloser-TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/9627444047fa4fe2865f20f167afcf0d to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/9627444047fa4fe2865f20f167afcf0d 2024-11-10T03:16:40,551 DEBUG [StoreCloser-TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/c70b7d9bc2614bb985329243a5c9ae0f to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/c70b7d9bc2614bb985329243a5c9ae0f 2024-11-10T03:16:40,552 DEBUG [StoreCloser-TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/acfca18bf378472cadd8637388d72a74 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/acfca18bf378472cadd8637388d72a74 2024-11-10T03:16:40,554 DEBUG [StoreCloser-TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/852f97728a4e49d18f38f2da39fd4ba2 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/852f97728a4e49d18f38f2da39fd4ba2 2024-11-10T03:16:40,555 DEBUG [StoreCloser-TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/239578cf7394453b807f66cc0d7133cb to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/239578cf7394453b807f66cc0d7133cb 2024-11-10T03:16:40,556 DEBUG [StoreCloser-TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/d12a9c8d0b4042ec9d3dc069b11b56a8 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/d12a9c8d0b4042ec9d3dc069b11b56a8 2024-11-10T03:16:40,557 DEBUG [StoreCloser-TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/bddf2f14f2d24a3ea6fa9994e7e6199a to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/bddf2f14f2d24a3ea6fa9994e7e6199a 2024-11-10T03:16:40,558 DEBUG [StoreCloser-TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/e4c7aceabb9d4b6d9ec6ece931eba970 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/e4c7aceabb9d4b6d9ec6ece931eba970 2024-11-10T03:16:40,559 DEBUG [StoreCloser-TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/71f31734b5b14fe7ba1b946619380fad to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/71f31734b5b14fe7ba1b946619380fad 2024-11-10T03:16:40,560 DEBUG [StoreCloser-TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/ed4f506be2e849638e5e2c1501cd0d40 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/ed4f506be2e849638e5e2c1501cd0d40 2024-11-10T03:16:40,561 DEBUG [StoreCloser-TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/5341143ae1c3426aa28c71e4527585c3 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/5341143ae1c3426aa28c71e4527585c3 2024-11-10T03:16:40,562 DEBUG [StoreCloser-TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/50195dcec2904f42833edd14a5f180ad to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/50195dcec2904f42833edd14a5f180ad 2024-11-10T03:16:40,563 DEBUG [StoreCloser-TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/4c1ecc6e66a3480391ba74e668ee873d to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/4c1ecc6e66a3480391ba74e668ee873d 2024-11-10T03:16:40,564 DEBUG [StoreCloser-TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/38a7848ea2a64fe8951b74af9b96f014 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/38a7848ea2a64fe8951b74af9b96f014 2024-11-10T03:16:40,566 DEBUG [StoreCloser-TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/5f62f3d4e99e4491a40b85e0ba9a7f37 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/5f62f3d4e99e4491a40b85e0ba9a7f37 2024-11-10T03:16:40,567 DEBUG [StoreCloser-TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/9a7c66c8aec8411493926b38ef391bc3 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/9a7c66c8aec8411493926b38ef391bc3 2024-11-10T03:16:40,568 DEBUG [StoreCloser-TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/85a2ab167dbf4767bc1bc36fd43ea015 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/85a2ab167dbf4767bc1bc36fd43ea015 2024-11-10T03:16:40,569 DEBUG [StoreCloser-TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/5028a17b18c144d7a38e4721f6db56cf to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/5028a17b18c144d7a38e4721f6db56cf 2024-11-10T03:16:40,571 DEBUG [StoreCloser-TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/fc5c4014748545bbad2d52631261bf32 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/fc5c4014748545bbad2d52631261bf32 2024-11-10T03:16:40,572 DEBUG [StoreCloser-TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/c9af7c0e745b4026a6c165f0d7d1c039 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/c9af7c0e745b4026a6c165f0d7d1c039 2024-11-10T03:16:40,573 DEBUG [StoreCloser-TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/fdb6f80b87f046d999c3fcc04a0c7092 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/fdb6f80b87f046d999c3fcc04a0c7092 2024-11-10T03:16:40,574 DEBUG [StoreCloser-TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/b0dc1941b0654086a68ba91c104a3d15 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/b0dc1941b0654086a68ba91c104a3d15 2024-11-10T03:16:40,575 DEBUG [StoreCloser-TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/56e26afc7ed54d2382b3210a9f7c54e2 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/56e26afc7ed54d2382b3210a9f7c54e2 2024-11-10T03:16:40,577 DEBUG [StoreCloser-TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/6c8489ed907444c2bb9ab7083aa9356d to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/6c8489ed907444c2bb9ab7083aa9356d 2024-11-10T03:16:40,578 DEBUG [StoreCloser-TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/f79000458af44e5c9770b1f5f651e383, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/a70057a05b2547d8a434878573a74db6, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/c857d14211d94a209f81c919a922c5e5, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/6f1248b6174342508ee260e35805ff00, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/9636f7c4fef04615b25eac6614ad11b2, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/dd40da2f209348c9b4fb55c6165fd802, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/5cde76b584874ebbbc85d3b6e9e02330, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/c8a45b47e177497aafa40a0f7c8d48a3, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/dbd12687927e4e4f9955c5defbae5203, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/a9daa8f6c68044c4865965f796c44c9e, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/18b79e975d43438d9ed669fbf4c38c3b, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/3c64207948624446bc587a1dc3b053a6, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/3f00d2151e6c417c9815126b19d2de00, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/64c4f6aee6f54bb9867f3eecfa40cdcb, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/834cd659a4924f78a0fafc8f0f89c777, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/c05ee47056fc4a1aab3294f34bf6271e, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/83b07bfbc4424aecaa19b9afd9ad17b6, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/e7797116da52449fa8540dc9dbdfda81, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/186e0006b1cb4782bdeae8fdc44d0e18, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/04ea8793b81e4692bc6d285de64e07c9, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/384fc2a7ac634580ba937629f9a7f372, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/67c5ef171591440198f81869accd5790, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/0daba8dbe18446b38f257f6d44a30d2d, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/49c70c22bf1e4a15a27d5c8c7b7137ee, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/0306fb6329ab45e7a5c4f0a881dd3055, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/e8310040b8b1444fbc34bc140eed3750, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/cb5bcfbbc6a846cda8034e740df56458, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/05f182cc6b2e46ecbde195e9994c0226, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/7278817c59de40bcaddf47a2d0140b1e, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/2e5acc6f0cc4442282aa61d9f70ce988, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/6f9c637a891f492fb081b088e0fd61ca, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/ad0383fa138f4d0f9ff8684dd9568a48] to archive 2024-11-10T03:16:40,579 DEBUG [StoreCloser-TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T03:16:40,582 DEBUG [StoreCloser-TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/f79000458af44e5c9770b1f5f651e383 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/f79000458af44e5c9770b1f5f651e383 2024-11-10T03:16:40,586 DEBUG [StoreCloser-TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/a70057a05b2547d8a434878573a74db6 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/a70057a05b2547d8a434878573a74db6 2024-11-10T03:16:40,587 DEBUG [StoreCloser-TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/c857d14211d94a209f81c919a922c5e5 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/c857d14211d94a209f81c919a922c5e5 2024-11-10T03:16:40,588 DEBUG [StoreCloser-TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/6f1248b6174342508ee260e35805ff00 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/6f1248b6174342508ee260e35805ff00 2024-11-10T03:16:40,589 DEBUG [StoreCloser-TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/9636f7c4fef04615b25eac6614ad11b2 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/9636f7c4fef04615b25eac6614ad11b2 2024-11-10T03:16:40,591 DEBUG [StoreCloser-TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/dd40da2f209348c9b4fb55c6165fd802 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/dd40da2f209348c9b4fb55c6165fd802 2024-11-10T03:16:40,592 DEBUG [StoreCloser-TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/5cde76b584874ebbbc85d3b6e9e02330 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/5cde76b584874ebbbc85d3b6e9e02330 2024-11-10T03:16:40,593 DEBUG [StoreCloser-TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/c8a45b47e177497aafa40a0f7c8d48a3 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/c8a45b47e177497aafa40a0f7c8d48a3 2024-11-10T03:16:40,594 DEBUG [StoreCloser-TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/dbd12687927e4e4f9955c5defbae5203 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/dbd12687927e4e4f9955c5defbae5203 2024-11-10T03:16:40,596 DEBUG [StoreCloser-TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/a9daa8f6c68044c4865965f796c44c9e to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/a9daa8f6c68044c4865965f796c44c9e 2024-11-10T03:16:40,598 DEBUG [StoreCloser-TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/18b79e975d43438d9ed669fbf4c38c3b to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/18b79e975d43438d9ed669fbf4c38c3b 2024-11-10T03:16:40,599 DEBUG [StoreCloser-TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/3c64207948624446bc587a1dc3b053a6 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/3c64207948624446bc587a1dc3b053a6 2024-11-10T03:16:40,600 DEBUG [StoreCloser-TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/3f00d2151e6c417c9815126b19d2de00 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/3f00d2151e6c417c9815126b19d2de00 2024-11-10T03:16:40,602 DEBUG [StoreCloser-TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/64c4f6aee6f54bb9867f3eecfa40cdcb to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/64c4f6aee6f54bb9867f3eecfa40cdcb 2024-11-10T03:16:40,603 DEBUG [StoreCloser-TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/834cd659a4924f78a0fafc8f0f89c777 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/834cd659a4924f78a0fafc8f0f89c777 2024-11-10T03:16:40,605 DEBUG [StoreCloser-TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/c05ee47056fc4a1aab3294f34bf6271e to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/c05ee47056fc4a1aab3294f34bf6271e 2024-11-10T03:16:40,606 DEBUG [StoreCloser-TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/83b07bfbc4424aecaa19b9afd9ad17b6 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/83b07bfbc4424aecaa19b9afd9ad17b6 2024-11-10T03:16:40,607 DEBUG [StoreCloser-TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/e7797116da52449fa8540dc9dbdfda81 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/e7797116da52449fa8540dc9dbdfda81 2024-11-10T03:16:40,608 DEBUG [StoreCloser-TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/186e0006b1cb4782bdeae8fdc44d0e18 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/186e0006b1cb4782bdeae8fdc44d0e18 2024-11-10T03:16:40,610 DEBUG [StoreCloser-TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/04ea8793b81e4692bc6d285de64e07c9 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/04ea8793b81e4692bc6d285de64e07c9 2024-11-10T03:16:40,611 DEBUG [StoreCloser-TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/384fc2a7ac634580ba937629f9a7f372 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/384fc2a7ac634580ba937629f9a7f372 2024-11-10T03:16:40,613 DEBUG [StoreCloser-TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/67c5ef171591440198f81869accd5790 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/67c5ef171591440198f81869accd5790 2024-11-10T03:16:40,614 DEBUG [StoreCloser-TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/0daba8dbe18446b38f257f6d44a30d2d to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/0daba8dbe18446b38f257f6d44a30d2d 2024-11-10T03:16:40,616 DEBUG [StoreCloser-TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/49c70c22bf1e4a15a27d5c8c7b7137ee to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/49c70c22bf1e4a15a27d5c8c7b7137ee 2024-11-10T03:16:40,617 DEBUG [StoreCloser-TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/0306fb6329ab45e7a5c4f0a881dd3055 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/0306fb6329ab45e7a5c4f0a881dd3055 2024-11-10T03:16:40,619 DEBUG [StoreCloser-TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/e8310040b8b1444fbc34bc140eed3750 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/e8310040b8b1444fbc34bc140eed3750 2024-11-10T03:16:40,620 DEBUG [StoreCloser-TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/cb5bcfbbc6a846cda8034e740df56458 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/cb5bcfbbc6a846cda8034e740df56458 2024-11-10T03:16:40,621 DEBUG [StoreCloser-TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/05f182cc6b2e46ecbde195e9994c0226 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/05f182cc6b2e46ecbde195e9994c0226 2024-11-10T03:16:40,623 DEBUG [StoreCloser-TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/7278817c59de40bcaddf47a2d0140b1e to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/7278817c59de40bcaddf47a2d0140b1e 2024-11-10T03:16:40,624 DEBUG [StoreCloser-TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/2e5acc6f0cc4442282aa61d9f70ce988 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/2e5acc6f0cc4442282aa61d9f70ce988 2024-11-10T03:16:40,625 DEBUG [StoreCloser-TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/6f9c637a891f492fb081b088e0fd61ca to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/6f9c637a891f492fb081b088e0fd61ca 2024-11-10T03:16:40,626 DEBUG [StoreCloser-TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/ad0383fa138f4d0f9ff8684dd9568a48 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/ad0383fa138f4d0f9ff8684dd9568a48 2024-11-10T03:16:40,632 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/recovered.edits/571.seqid, newMaxSeqId=571, maxSeqId=1 2024-11-10T03:16:40,633 INFO [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. 2024-11-10T03:16:40,634 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1676): Region close journal for 77ed36df592326523261438e65e5b722: Waiting for close lock at 1731208599127Running coprocessor pre-close hooks at 1731208599127Disabling compacts and flushes for region at 1731208599127Disabling writes for close at 1731208599127Obtaining lock to block concurrent updates at 1731208599128 (+1 ms)Preparing flush snapshotting stores in 77ed36df592326523261438e65e5b722 at 1731208599128Finished memstore snapshotting TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1731208599128Flushing stores of TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722. at 1731208599129 (+1 ms)Flushing 77ed36df592326523261438e65e5b722/A: creating writer at 1731208599129Flushing 77ed36df592326523261438e65e5b722/A: appending metadata at 1731208599193 (+64 ms)Flushing 77ed36df592326523261438e65e5b722/A: closing flushed file at 1731208599193Flushing 77ed36df592326523261438e65e5b722/B: creating writer at 1731208599610 (+417 ms)Flushing 77ed36df592326523261438e65e5b722/B: appending metadata at 1731208599612 (+2 ms)Flushing 77ed36df592326523261438e65e5b722/B: closing flushed file at 1731208599613 (+1 ms)Flushing 77ed36df592326523261438e65e5b722/C: creating writer at 1731208600023 (+410 ms)Flushing 77ed36df592326523261438e65e5b722/C: appending metadata at 1731208600025 (+2 ms)Flushing 77ed36df592326523261438e65e5b722/C: closing flushed file at 1731208600025Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@73befd30: reopening flushed file at 1731208600435 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3b702773: reopening flushed file at 1731208600443 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5f96876a: reopening flushed file at 1731208600459 (+16 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 77ed36df592326523261438e65e5b722 in 1340ms, sequenceid=568, compaction requested=false at 1731208600467 (+8 ms)Writing region close event to WAL at 1731208600628 (+161 ms)Running coprocessor post-close hooks at 1731208600633 (+5 ms)Closed at 1731208600633 2024-11-10T03:16:40,637 INFO [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] handler.UnassignRegionHandler(157): Closed 77ed36df592326523261438e65e5b722 2024-11-10T03:16:40,637 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=77ed36df592326523261438e65e5b722, regionState=CLOSED 2024-11-10T03:16:40,640 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=25, state=RUNNABLE, hasLock=false; CloseRegionProcedure 77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 because future has completed 2024-11-10T03:16:40,644 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-10T03:16:40,644 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; CloseRegionProcedure 77ed36df592326523261438e65e5b722, server=394d2222f724,38139,1731208572054 in 1.6730 sec 2024-11-10T03:16:40,647 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=25, resume processing ppid=24 2024-11-10T03:16:40,647 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, ppid=24, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=77ed36df592326523261438e65e5b722, UNASSIGN in 1.6810 sec 2024-11-10T03:16:40,654 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-10T03:16:40,655 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.6910 sec 2024-11-10T03:16:40,655 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731208600654"}]},"ts":"1731208600654"} 2024-11-10T03:16:40,657 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-10T03:16:40,657 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-10T03:16:40,661 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.7100 sec 2024-11-10T03:16:41,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-10T03:16:41,104 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-10T03:16:41,106 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-10T03:16:41,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T03:16:41,110 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=27, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T03:16:41,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-10T03:16:41,112 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=27, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T03:16:41,116 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722 2024-11-10T03:16:41,121 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A, FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B, FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C, FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/recovered.edits] 2024-11-10T03:16:41,125 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/89ffb4f7e0b64e918b8fb0f761983762 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/89ffb4f7e0b64e918b8fb0f761983762 2024-11-10T03:16:41,126 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/cbb970cf5767461f85160b19057c95fc to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/A/cbb970cf5767461f85160b19057c95fc 2024-11-10T03:16:41,129 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/7f5e84ac091e4713b3f6c790525b3f0c to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/7f5e84ac091e4713b3f6c790525b3f0c 2024-11-10T03:16:41,130 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/a21065c06a3945fe94263440f718980a to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/B/a21065c06a3945fe94263440f718980a 2024-11-10T03:16:41,133 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/046b7e6bf0a14e8e952f9b5d8542e89d to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/046b7e6bf0a14e8e952f9b5d8542e89d 2024-11-10T03:16:41,134 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/cb0a9f36e98c487b985bacec3d346df0 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/C/cb0a9f36e98c487b985bacec3d346df0 2024-11-10T03:16:41,137 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/recovered.edits/571.seqid to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722/recovered.edits/571.seqid 2024-11-10T03:16:41,137 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/77ed36df592326523261438e65e5b722 2024-11-10T03:16:41,137 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-10T03:16:41,141 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=27, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T03:16:41,144 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-10T03:16:41,150 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-10T03:16:41,181 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-10T03:16:41,182 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=27, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T03:16:41,183 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-10T03:16:41,183 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731208601183"}]},"ts":"9223372036854775807"} 2024-11-10T03:16:41,187 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-10T03:16:41,187 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 77ed36df592326523261438e65e5b722, NAME => 'TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722.', STARTKEY => '', ENDKEY => ''}] 2024-11-10T03:16:41,187 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-10T03:16:41,187 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731208601187"}]},"ts":"9223372036854775807"} 2024-11-10T03:16:41,190 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-10T03:16:41,191 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=27, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T03:16:41,193 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 85 msec 2024-11-10T03:16:41,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-10T03:16:41,224 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-10T03:16:41,224 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-10T03:16:41,233 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=250 (was 211) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1264325177_22 at /127.0.0.1:47376 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/394d2222f724:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-972336514_22 at /127.0.0.1:49068 [Waiting for operation #318] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;394d2222f724:38139-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-972336514_22 at /127.0.0.1:47364 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1264325177_22 at /127.0.0.1:49198 [Waiting for operation #300] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/394d2222f724:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=457 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=769 (was 686) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2753 (was 3361) 2024-11-10T03:16:41,245 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=250, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=769, ProcessCount=11, AvailableMemoryMB=2750 2024-11-10T03:16:41,246 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-10T03:16:41,247 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T03:16:41,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] procedure2.ProcedureExecutor(1139): Stored pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-10T03:16:41,250 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-10T03:16:41,250 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:41,250 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 28 2024-11-10T03:16:41,251 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-10T03:16:41,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-10T03:16:41,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741939_1115 (size=1061) 2024-11-10T03:16:41,267 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8 2024-11-10T03:16:41,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741940_1116 (size=53) 2024-11-10T03:16:41,283 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T03:16:41,283 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing f294ea12149d12df042ace2dfb55c011, disabling compactions & flushes 2024-11-10T03:16:41,283 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:16:41,283 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:16:41,283 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. after waiting 0 ms 2024-11-10T03:16:41,283 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:16:41,283 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:16:41,284 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for f294ea12149d12df042ace2dfb55c011: Waiting for close lock at 1731208601283Disabling compacts and flushes for region at 1731208601283Disabling writes for close at 1731208601283Writing region close event to WAL at 1731208601283Closed at 1731208601283 2024-11-10T03:16:41,285 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-10T03:16:41,285 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731208601285"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731208601285"}]},"ts":"1731208601285"} 2024-11-10T03:16:41,287 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-10T03:16:41,288 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-10T03:16:41,289 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731208601289"}]},"ts":"1731208601289"} 2024-11-10T03:16:41,291 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-10T03:16:41,291 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f294ea12149d12df042ace2dfb55c011, ASSIGN}] 2024-11-10T03:16:41,293 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f294ea12149d12df042ace2dfb55c011, ASSIGN 2024-11-10T03:16:41,296 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f294ea12149d12df042ace2dfb55c011, ASSIGN; state=OFFLINE, location=394d2222f724,38139,1731208572054; forceNewPlan=false, retain=false 2024-11-10T03:16:41,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-10T03:16:41,447 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=f294ea12149d12df042ace2dfb55c011, regionState=OPENING, regionLocation=394d2222f724,38139,1731208572054 2024-11-10T03:16:41,450 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f294ea12149d12df042ace2dfb55c011, ASSIGN because future has completed 2024-11-10T03:16:41,451 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054}] 2024-11-10T03:16:41,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-10T03:16:41,608 INFO [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:16:41,608 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7752): Opening region: {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} 2024-11-10T03:16:41,608 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees f294ea12149d12df042ace2dfb55c011 2024-11-10T03:16:41,609 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T03:16:41,609 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7794): checking encryption for f294ea12149d12df042ace2dfb55c011 2024-11-10T03:16:41,609 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7797): checking classloading for f294ea12149d12df042ace2dfb55c011 2024-11-10T03:16:41,610 INFO [StoreOpener-f294ea12149d12df042ace2dfb55c011-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region f294ea12149d12df042ace2dfb55c011 2024-11-10T03:16:41,611 INFO [StoreOpener-f294ea12149d12df042ace2dfb55c011-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T03:16:41,612 INFO [StoreOpener-f294ea12149d12df042ace2dfb55c011-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f294ea12149d12df042ace2dfb55c011 columnFamilyName A 2024-11-10T03:16:41,612 DEBUG [StoreOpener-f294ea12149d12df042ace2dfb55c011-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:41,613 INFO [StoreOpener-f294ea12149d12df042ace2dfb55c011-1 {}] regionserver.HStore(327): Store=f294ea12149d12df042ace2dfb55c011/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T03:16:41,613 INFO [StoreOpener-f294ea12149d12df042ace2dfb55c011-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region f294ea12149d12df042ace2dfb55c011 2024-11-10T03:16:41,614 INFO [StoreOpener-f294ea12149d12df042ace2dfb55c011-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T03:16:41,614 INFO [StoreOpener-f294ea12149d12df042ace2dfb55c011-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f294ea12149d12df042ace2dfb55c011 columnFamilyName B 2024-11-10T03:16:41,614 DEBUG [StoreOpener-f294ea12149d12df042ace2dfb55c011-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:41,614 INFO [StoreOpener-f294ea12149d12df042ace2dfb55c011-1 {}] regionserver.HStore(327): Store=f294ea12149d12df042ace2dfb55c011/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T03:16:41,614 INFO [StoreOpener-f294ea12149d12df042ace2dfb55c011-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region f294ea12149d12df042ace2dfb55c011 2024-11-10T03:16:41,615 INFO [StoreOpener-f294ea12149d12df042ace2dfb55c011-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T03:16:41,615 INFO [StoreOpener-f294ea12149d12df042ace2dfb55c011-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f294ea12149d12df042ace2dfb55c011 columnFamilyName C 2024-11-10T03:16:41,616 DEBUG [StoreOpener-f294ea12149d12df042ace2dfb55c011-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:41,616 INFO [StoreOpener-f294ea12149d12df042ace2dfb55c011-1 {}] regionserver.HStore(327): Store=f294ea12149d12df042ace2dfb55c011/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T03:16:41,616 INFO [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:16:41,616 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1038): replaying wal for f294ea12149d12df042ace2dfb55c011 2024-11-10T03:16:41,617 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011 2024-11-10T03:16:41,617 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011 2024-11-10T03:16:41,618 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1048): stopping wal replay for f294ea12149d12df042ace2dfb55c011 2024-11-10T03:16:41,618 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1060): Cleaning up temporary data for f294ea12149d12df042ace2dfb55c011 2024-11-10T03:16:41,618 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T03:16:41,619 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1093): writing seq id for f294ea12149d12df042ace2dfb55c011 2024-11-10T03:16:41,621 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T03:16:41,622 INFO [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1114): Opened f294ea12149d12df042ace2dfb55c011; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61097261, jitterRate=-0.08957986533641815}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T03:16:41,622 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f294ea12149d12df042ace2dfb55c011 2024-11-10T03:16:41,623 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1006): Region open journal for f294ea12149d12df042ace2dfb55c011: Running coprocessor pre-open hook at 1731208601609Writing region info on filesystem at 1731208601609Initializing all the Stores at 1731208601610 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731208601610Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731208601610Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731208601610Cleaning up temporary data from old regions at 1731208601618 (+8 ms)Running coprocessor post-open hooks at 1731208601622 (+4 ms)Region opened successfully at 1731208601623 (+1 ms) 2024-11-10T03:16:41,624 INFO [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., pid=30, masterSystemTime=1731208601605 2024-11-10T03:16:41,626 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:16:41,626 INFO [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:16:41,626 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=f294ea12149d12df042ace2dfb55c011, regionState=OPEN, openSeqNum=2, regionLocation=394d2222f724,38139,1731208572054 2024-11-10T03:16:41,628 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 because future has completed 2024-11-10T03:16:41,631 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-10T03:16:41,631 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 in 178 msec 2024-11-10T03:16:41,634 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-11-10T03:16:41,634 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f294ea12149d12df042ace2dfb55c011, ASSIGN in 340 msec 2024-11-10T03:16:41,635 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-10T03:16:41,635 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731208601635"}]},"ts":"1731208601635"} 2024-11-10T03:16:41,637 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-10T03:16:41,638 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-10T03:16:41,641 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 391 msec 2024-11-10T03:16:41,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-10T03:16:41,882 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-10T03:16:41,883 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d44c0b9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:16:41,883 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 394d2222f724,37299,-1 for getting cluster id 2024-11-10T03:16:41,884 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T03:16:41,887 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f5c6727-65f7-4659-afe8-8e9a55664c78' 2024-11-10T03:16:41,888 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T03:16:41,888 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f5c6727-65f7-4659-afe8-8e9a55664c78" 2024-11-10T03:16:41,889 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@356b85f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:16:41,889 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [394d2222f724,37299,-1] 2024-11-10T03:16:41,889 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T03:16:41,889 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:16:41,899 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49040, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T03:16:41,901 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4474113, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:16:41,901 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T03:16:41,903 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=394d2222f724,38139,1731208572054, seqNum=-1] 2024-11-10T03:16:41,904 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T03:16:41,906 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37874, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T03:16:41,909 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-10T03:16:41,910 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 394d2222f724,37299,1731208571229 2024-11-10T03:16:41,910 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@22d64e14 2024-11-10T03:16:41,911 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-10T03:16:41,913 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49044, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-10T03:16:41,920 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-10T03:16:41,921 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T03:16:41,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-10T03:16:41,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741941_1117 (size=1097) 2024-11-10T03:16:42,381 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-10T03:16:42,381 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-10T03:16:42,387 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-10T03:16:42,398 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f294ea12149d12df042ace2dfb55c011, REOPEN/MOVE}] 2024-11-10T03:16:42,400 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f294ea12149d12df042ace2dfb55c011, REOPEN/MOVE 2024-11-10T03:16:42,401 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=f294ea12149d12df042ace2dfb55c011, regionState=CLOSING, regionLocation=394d2222f724,38139,1731208572054 2024-11-10T03:16:42,409 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f294ea12149d12df042ace2dfb55c011, REOPEN/MOVE because future has completed 2024-11-10T03:16:42,410 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-10T03:16:42,410 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054}] 2024-11-10T03:16:42,566 INFO [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(122): Close f294ea12149d12df042ace2dfb55c011 2024-11-10T03:16:42,566 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-10T03:16:42,566 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1722): Closing f294ea12149d12df042ace2dfb55c011, disabling compactions & flushes 2024-11-10T03:16:42,566 INFO [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:16:42,566 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:16:42,566 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. after waiting 0 ms 2024-11-10T03:16:42,566 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:16:42,589 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-10T03:16:42,593 INFO [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:16:42,593 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1676): Region close journal for f294ea12149d12df042ace2dfb55c011: Waiting for close lock at 1731208602566Running coprocessor pre-close hooks at 1731208602566Disabling compacts and flushes for region at 1731208602566Disabling writes for close at 1731208602566Writing region close event to WAL at 1731208602569 (+3 ms)Running coprocessor post-close hooks at 1731208602593 (+24 ms)Closed at 1731208602593 2024-11-10T03:16:42,593 WARN [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegionServer(3299): Not adding moved region record: f294ea12149d12df042ace2dfb55c011 to self. 2024-11-10T03:16:42,598 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=f294ea12149d12df042ace2dfb55c011, regionState=CLOSED 2024-11-10T03:16:42,601 INFO [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(157): Closed f294ea12149d12df042ace2dfb55c011 2024-11-10T03:16:42,603 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 because future has completed 2024-11-10T03:16:42,614 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-10T03:16:42,615 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 in 200 msec 2024-11-10T03:16:42,616 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f294ea12149d12df042ace2dfb55c011, REOPEN/MOVE; state=CLOSED, location=394d2222f724,38139,1731208572054; forceNewPlan=false, retain=true 2024-11-10T03:16:42,766 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=f294ea12149d12df042ace2dfb55c011, regionState=OPENING, regionLocation=394d2222f724,38139,1731208572054 2024-11-10T03:16:42,769 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f294ea12149d12df042ace2dfb55c011, REOPEN/MOVE because future has completed 2024-11-10T03:16:42,769 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054}] 2024-11-10T03:16:42,925 INFO [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:16:42,925 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7752): Opening region: {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} 2024-11-10T03:16:42,925 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees f294ea12149d12df042ace2dfb55c011 2024-11-10T03:16:42,925 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T03:16:42,926 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7794): checking encryption for f294ea12149d12df042ace2dfb55c011 2024-11-10T03:16:42,926 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7797): checking classloading for f294ea12149d12df042ace2dfb55c011 2024-11-10T03:16:42,929 INFO [StoreOpener-f294ea12149d12df042ace2dfb55c011-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region f294ea12149d12df042ace2dfb55c011 2024-11-10T03:16:42,930 INFO [StoreOpener-f294ea12149d12df042ace2dfb55c011-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T03:16:42,935 INFO [StoreOpener-f294ea12149d12df042ace2dfb55c011-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f294ea12149d12df042ace2dfb55c011 columnFamilyName A 2024-11-10T03:16:42,938 DEBUG [StoreOpener-f294ea12149d12df042ace2dfb55c011-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:42,939 INFO [StoreOpener-f294ea12149d12df042ace2dfb55c011-1 {}] regionserver.HStore(327): Store=f294ea12149d12df042ace2dfb55c011/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T03:16:42,940 INFO [StoreOpener-f294ea12149d12df042ace2dfb55c011-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region f294ea12149d12df042ace2dfb55c011 2024-11-10T03:16:42,941 INFO [StoreOpener-f294ea12149d12df042ace2dfb55c011-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T03:16:42,941 INFO [StoreOpener-f294ea12149d12df042ace2dfb55c011-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f294ea12149d12df042ace2dfb55c011 columnFamilyName B 2024-11-10T03:16:42,941 DEBUG [StoreOpener-f294ea12149d12df042ace2dfb55c011-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:42,942 INFO [StoreOpener-f294ea12149d12df042ace2dfb55c011-1 {}] regionserver.HStore(327): Store=f294ea12149d12df042ace2dfb55c011/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T03:16:42,942 INFO [StoreOpener-f294ea12149d12df042ace2dfb55c011-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region f294ea12149d12df042ace2dfb55c011 2024-11-10T03:16:42,943 INFO [StoreOpener-f294ea12149d12df042ace2dfb55c011-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T03:16:42,943 INFO [StoreOpener-f294ea12149d12df042ace2dfb55c011-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f294ea12149d12df042ace2dfb55c011 columnFamilyName C 2024-11-10T03:16:42,943 DEBUG [StoreOpener-f294ea12149d12df042ace2dfb55c011-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:42,944 INFO [StoreOpener-f294ea12149d12df042ace2dfb55c011-1 {}] regionserver.HStore(327): Store=f294ea12149d12df042ace2dfb55c011/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T03:16:42,944 INFO [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:16:42,944 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1038): replaying wal for f294ea12149d12df042ace2dfb55c011 2024-11-10T03:16:42,945 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011 2024-11-10T03:16:42,946 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011 2024-11-10T03:16:42,947 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1048): stopping wal replay for f294ea12149d12df042ace2dfb55c011 2024-11-10T03:16:42,947 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1060): Cleaning up temporary data for f294ea12149d12df042ace2dfb55c011 2024-11-10T03:16:42,948 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T03:16:42,949 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1093): writing seq id for f294ea12149d12df042ace2dfb55c011 2024-11-10T03:16:42,950 INFO [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1114): Opened f294ea12149d12df042ace2dfb55c011; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66167309, jitterRate=-0.01403026282787323}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T03:16:42,950 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f294ea12149d12df042ace2dfb55c011 2024-11-10T03:16:42,951 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1006): Region open journal for f294ea12149d12df042ace2dfb55c011: Running coprocessor pre-open hook at 1731208602926Writing region info on filesystem at 1731208602926Initializing all the Stores at 1731208602927 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731208602927Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731208602927Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731208602927Cleaning up temporary data from old regions at 1731208602947 (+20 ms)Running coprocessor post-open hooks at 1731208602950 (+3 ms)Region opened successfully at 1731208602951 (+1 ms) 2024-11-10T03:16:42,951 INFO [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., pid=35, masterSystemTime=1731208602921 2024-11-10T03:16:42,953 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:16:42,953 INFO [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:16:42,954 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=f294ea12149d12df042ace2dfb55c011, regionState=OPEN, openSeqNum=5, regionLocation=394d2222f724,38139,1731208572054 2024-11-10T03:16:42,955 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 because future has completed 2024-11-10T03:16:42,957 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=33 2024-11-10T03:16:42,957 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=33, state=SUCCESS, hasLock=false; OpenRegionProcedure f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 in 187 msec 2024-11-10T03:16:42,959 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-10T03:16:42,959 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f294ea12149d12df042ace2dfb55c011, REOPEN/MOVE in 560 msec 2024-11-10T03:16:42,962 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-10T03:16:42,962 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 574 msec 2024-11-10T03:16:42,965 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 1.0400 sec 2024-11-10T03:16:42,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-10T03:16:42,967 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-10T03:16:42,971 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4eb7e9dc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:16:42,971 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 394d2222f724,37299,-1 for getting cluster id 2024-11-10T03:16:42,971 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T03:16:42,973 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f5c6727-65f7-4659-afe8-8e9a55664c78' 2024-11-10T03:16:42,973 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T03:16:42,973 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f5c6727-65f7-4659-afe8-8e9a55664c78" 2024-11-10T03:16:42,973 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ba0b500, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:16:42,973 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [394d2222f724,37299,-1] 2024-11-10T03:16:42,974 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T03:16:42,974 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:16:42,975 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49064, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T03:16:42,976 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78276902, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:16:42,977 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@486e0901, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:16:42,977 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 394d2222f724,37299,-1 for getting cluster id 2024-11-10T03:16:42,978 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T03:16:42,978 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f5c6727-65f7-4659-afe8-8e9a55664c78' 2024-11-10T03:16:42,979 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T03:16:42,979 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f5c6727-65f7-4659-afe8-8e9a55664c78" 2024-11-10T03:16:42,979 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@579718f6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:16:42,979 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [394d2222f724,37299,-1] 2024-11-10T03:16:42,979 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T03:16:42,980 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:16:42,981 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49080, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T03:16:42,981 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2748991c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:16:42,982 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42f18193, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:16:42,982 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 394d2222f724,37299,-1 for getting cluster id 2024-11-10T03:16:42,982 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T03:16:42,983 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f5c6727-65f7-4659-afe8-8e9a55664c78' 2024-11-10T03:16:42,983 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T03:16:42,983 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f5c6727-65f7-4659-afe8-8e9a55664c78" 2024-11-10T03:16:42,983 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18af1444, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:16:42,983 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [394d2222f724,37299,-1] 2024-11-10T03:16:42,984 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T03:16:42,984 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:16:42,985 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49086, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T03:16:42,985 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e9dfe2e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:16:42,986 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2cde9120, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:16:42,986 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 394d2222f724,37299,-1 for getting cluster id 2024-11-10T03:16:42,986 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T03:16:42,987 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f5c6727-65f7-4659-afe8-8e9a55664c78' 2024-11-10T03:16:42,987 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T03:16:42,987 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f5c6727-65f7-4659-afe8-8e9a55664c78" 2024-11-10T03:16:42,987 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22547ece, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:16:42,987 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [394d2222f724,37299,-1] 2024-11-10T03:16:42,988 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T03:16:42,988 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:16:42,989 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49104, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T03:16:42,989 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d1b7899, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:16:42,990 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a1653a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:16:42,990 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 394d2222f724,37299,-1 for getting cluster id 2024-11-10T03:16:42,990 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T03:16:42,991 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f5c6727-65f7-4659-afe8-8e9a55664c78' 2024-11-10T03:16:42,991 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T03:16:42,991 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f5c6727-65f7-4659-afe8-8e9a55664c78" 2024-11-10T03:16:42,991 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58b719ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:16:42,991 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [394d2222f724,37299,-1] 2024-11-10T03:16:42,991 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T03:16:42,992 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:16:42,992 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49120, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T03:16:42,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a5c4410, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:16:42,994 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48fa262a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:16:42,994 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 394d2222f724,37299,-1 for getting cluster id 2024-11-10T03:16:42,994 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T03:16:42,995 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f5c6727-65f7-4659-afe8-8e9a55664c78' 2024-11-10T03:16:42,995 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T03:16:42,995 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f5c6727-65f7-4659-afe8-8e9a55664c78" 2024-11-10T03:16:42,995 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22ab48ce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:16:42,995 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [394d2222f724,37299,-1] 2024-11-10T03:16:42,995 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T03:16:42,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:16:42,996 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49142, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T03:16:42,997 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4bb74346, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:16:42,998 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5808d603, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:16:42,998 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 394d2222f724,37299,-1 for getting cluster id 2024-11-10T03:16:42,998 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T03:16:42,998 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f5c6727-65f7-4659-afe8-8e9a55664c78' 2024-11-10T03:16:42,999 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T03:16:42,999 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f5c6727-65f7-4659-afe8-8e9a55664c78" 2024-11-10T03:16:42,999 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@560d502b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:16:42,999 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [394d2222f724,37299,-1] 2024-11-10T03:16:42,999 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T03:16:43,000 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:16:43,001 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49172, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T03:16:43,001 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34c0b19c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:16:43,002 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49c960a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:16:43,002 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 394d2222f724,37299,-1 for getting cluster id 2024-11-10T03:16:43,002 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T03:16:43,003 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f5c6727-65f7-4659-afe8-8e9a55664c78' 2024-11-10T03:16:43,003 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T03:16:43,003 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f5c6727-65f7-4659-afe8-8e9a55664c78" 2024-11-10T03:16:43,003 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2baaac90, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:16:43,003 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [394d2222f724,37299,-1] 2024-11-10T03:16:43,003 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T03:16:43,004 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:16:43,004 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49180, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T03:16:43,005 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1985fcae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:16:43,006 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c277cfb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:16:43,006 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 394d2222f724,37299,-1 for getting cluster id 2024-11-10T03:16:43,006 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T03:16:43,007 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f5c6727-65f7-4659-afe8-8e9a55664c78' 2024-11-10T03:16:43,007 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T03:16:43,007 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f5c6727-65f7-4659-afe8-8e9a55664c78" 2024-11-10T03:16:43,007 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fa29de0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:16:43,007 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [394d2222f724,37299,-1] 2024-11-10T03:16:43,008 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T03:16:43,008 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:16:43,009 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49200, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T03:16:43,009 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7405a174, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:16:43,010 DEBUG [Thread-611 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T03:16:43,010 DEBUG [Thread-609 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T03:16:43,010 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T03:16:43,011 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=394d2222f724,38139,1731208572054, seqNum=-1] 2024-11-10T03:16:43,011 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=394d2222f724,38139,1731208572054, seqNum=-1] 2024-11-10T03:16:43,012 DEBUG [Thread-604 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T03:16:43,012 DEBUG [Thread-600 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T03:16:43,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] procedure2.ProcedureExecutor(1139): Stored pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees 2024-11-10T03:16:43,012 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T03:16:43,013 DEBUG [Thread-600 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=394d2222f724,38139,1731208572054, seqNum=-1] 2024-11-10T03:16:43,013 DEBUG [Thread-600 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T03:16:43,013 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T03:16:43,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T03:16:43,013 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37886, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T03:16:43,014 DEBUG [Thread-604 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=394d2222f724,38139,1731208572054, seqNum=-1] 2024-11-10T03:16:43,014 DEBUG [Thread-604 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T03:16:43,014 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T03:16:43,014 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37896, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T03:16:43,014 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T03:16:43,014 DEBUG [Thread-615 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T03:16:43,015 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37902, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T03:16:43,015 DEBUG [Thread-602 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T03:16:43,015 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37914, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T03:16:43,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-10T03:16:43,017 DEBUG [Thread-613 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T03:16:43,017 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=394d2222f724,38139,1731208572054, seqNum=-1] 2024-11-10T03:16:43,017 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T03:16:43,018 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5] 2024-11-10T03:16:43,018 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=394d2222f724,38139,1731208572054, seqNum=-1] 2024-11-10T03:16:43,018 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T03:16:43,018 DEBUG [Thread-598 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T03:16:43,019 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37918, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T03:16:43,020 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5] 2024-11-10T03:16:43,020 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=394d2222f724,38139,1731208572054, seqNum=-1] 2024-11-10T03:16:43,020 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T03:16:43,021 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37932, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T03:16:43,021 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=394d2222f724,38139,1731208572054, seqNum=-1] 2024-11-10T03:16:43,021 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T03:16:43,022 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37948, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T03:16:43,022 DEBUG [Thread-606 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T03:16:43,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=394d2222f724,38139,1731208572054, seqNum=-1] 2024-11-10T03:16:43,023 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5] 2024-11-10T03:16:43,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T03:16:43,023 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37958, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T03:16:43,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5] 2024-11-10T03:16:43,024 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5] 2024-11-10T03:16:43,024 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5] 2024-11-10T03:16:43,024 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5] 2024-11-10T03:16:43,026 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37968, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T03:16:43,026 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5] 2024-11-10T03:16:43,029 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5] 2024-11-10T03:16:43,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on f294ea12149d12df042ace2dfb55c011 2024-11-10T03:16:43,051 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f294ea12149d12df042ace2dfb55c011 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-10T03:16:43,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f294ea12149d12df042ace2dfb55c011, store=A 2024-11-10T03:16:43,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:16:43,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f294ea12149d12df042ace2dfb55c011, store=B 2024-11-10T03:16:43,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:16:43,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f294ea12149d12df042ace2dfb55c011, store=C 2024-11-10T03:16:43,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:16:43,110 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111074cbef41c7434b308bab603bfcd3212a_f294ea12149d12df042ace2dfb55c011 is 50, key is test_row_0/A:col10/1731208603036/Put/seqid=0 2024-11-10T03:16:43,112 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:43,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37914 deadline: 1731208663095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:43,113 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:43,113 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:43,114 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:43,113 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:43,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37896 deadline: 1731208663097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:43,115 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:43,115 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:43,115 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:43,117 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:43,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37948 deadline: 1731208663099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:43,118 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:43,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37958 deadline: 1731208663101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:43,118 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:43,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:43,118 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:43,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:43,119 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:43,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:43,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:43,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37968 deadline: 1731208663104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:43,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:43,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:43,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:43,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-10T03:16:43,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741942_1118 (size=12154) 2024-11-10T03:16:43,169 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:43,169 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-10T03:16:43,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:16:43,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. as already flushing 2024-11-10T03:16:43,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:16:43,170 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:43,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:43,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:43,178 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111074cbef41c7434b308bab603bfcd3212a_f294ea12149d12df042ace2dfb55c011 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111074cbef41c7434b308bab603bfcd3212a_f294ea12149d12df042ace2dfb55c011 2024-11-10T03:16:43,179 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/A/336f3beea4234c409697878d7c1cf6e0, store: [table=TestAcidGuarantees family=A region=f294ea12149d12df042ace2dfb55c011] 2024-11-10T03:16:43,187 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/A/336f3beea4234c409697878d7c1cf6e0 is 175, key is test_row_0/A:col10/1731208603036/Put/seqid=0 2024-11-10T03:16:43,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741943_1119 (size=30955) 2024-11-10T03:16:43,227 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:43,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37914 deadline: 1731208663225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:43,227 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/A/336f3beea4234c409697878d7c1cf6e0 2024-11-10T03:16:43,227 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:43,227 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:43,227 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:43,227 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:43,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37896 deadline: 1731208663225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:43,228 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:43,228 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:43,228 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:43,229 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:43,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37958 deadline: 1731208663226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:43,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:43,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:43,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:43,230 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:43,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37948 deadline: 1731208663226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:43,231 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:43,231 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:43,231 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:43,231 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:43,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37968 deadline: 1731208663226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:43,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:43,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:43,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:43,287 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/B/213d7f1b07a94046a71959658c84ed06 is 50, key is test_row_0/B:col10/1731208603036/Put/seqid=0 2024-11-10T03:16:43,324 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-10T03:16:43,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:16:43,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. as already flushing 2024-11-10T03:16:43,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:16:43,325 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:43,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:43,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:43,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-10T03:16:43,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741944_1120 (size=12001) 2024-11-10T03:16:43,438 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:43,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37896 deadline: 1731208663434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:43,439 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:43,439 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:43,439 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:43,439 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:43,439 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:43,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37914 deadline: 1731208663434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:43,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37958 deadline: 1731208663434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:43,440 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:43,440 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:43,440 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:43,440 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:43,441 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:43,441 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:43,449 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:43,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37948 deadline: 1731208663443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:43,450 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:43,450 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:43,450 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:43,450 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:43,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37968 deadline: 1731208663443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:43,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:43,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:43,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:43,479 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-10T03:16:43,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:16:43,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. as already flushing 2024-11-10T03:16:43,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:16:43,486 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:43,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:43,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:43,648 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-10T03:16:43,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-10T03:16:43,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:16:43,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. as already flushing 2024-11-10T03:16:43,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:16:43,649 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:43,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:43,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:43,737 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/B/213d7f1b07a94046a71959658c84ed06 2024-11-10T03:16:43,749 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:43,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37896 deadline: 1731208663743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:43,751 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:43,753 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:43,754 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:43,762 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:43,762 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:43,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37958 deadline: 1731208663753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:43,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37914 deadline: 1731208663753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:43,764 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:43,764 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:43,764 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:43,764 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:43,764 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:43,765 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:43,777 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:43,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37948 deadline: 1731208663763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:43,778 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:43,778 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:43,778 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:43,778 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:43,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37968 deadline: 1731208663763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:43,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:43,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:43,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:43,798 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/C/3d2fc9f3c23846299a1f7aae62b47ae1 is 50, key is test_row_0/C:col10/1731208603036/Put/seqid=0 2024-11-10T03:16:43,804 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-10T03:16:43,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:16:43,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. as already flushing 2024-11-10T03:16:43,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:16:43,808 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:43,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:43,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:43,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741945_1121 (size=12001) 2024-11-10T03:16:43,844 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/C/3d2fc9f3c23846299a1f7aae62b47ae1 2024-11-10T03:16:43,868 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/A/336f3beea4234c409697878d7c1cf6e0 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/336f3beea4234c409697878d7c1cf6e0 2024-11-10T03:16:43,875 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/336f3beea4234c409697878d7c1cf6e0, entries=150, sequenceid=16, filesize=30.2 K 2024-11-10T03:16:43,877 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/B/213d7f1b07a94046a71959658c84ed06 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/213d7f1b07a94046a71959658c84ed06 2024-11-10T03:16:43,898 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/213d7f1b07a94046a71959658c84ed06, entries=150, sequenceid=16, filesize=11.7 K 2024-11-10T03:16:43,905 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/C/3d2fc9f3c23846299a1f7aae62b47ae1 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/3d2fc9f3c23846299a1f7aae62b47ae1 2024-11-10T03:16:43,913 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/3d2fc9f3c23846299a1f7aae62b47ae1, entries=150, sequenceid=16, filesize=11.7 K 2024-11-10T03:16:43,915 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for f294ea12149d12df042ace2dfb55c011 in 865ms, sequenceid=16, compaction requested=false 2024-11-10T03:16:43,915 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f294ea12149d12df042ace2dfb55c011: 2024-11-10T03:16:43,969 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-10T03:16:43,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:16:43,970 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2902): Flushing f294ea12149d12df042ace2dfb55c011 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-10T03:16:43,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f294ea12149d12df042ace2dfb55c011, store=A 2024-11-10T03:16:43,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:16:43,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f294ea12149d12df042ace2dfb55c011, store=B 2024-11-10T03:16:43,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:16:43,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f294ea12149d12df042ace2dfb55c011, store=C 2024-11-10T03:16:43,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:16:43,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111099cd0c1f927d471988d0ffeeb5844ff3_f294ea12149d12df042ace2dfb55c011 is 50, key is test_row_0/A:col10/1731208603092/Put/seqid=0 2024-11-10T03:16:43,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741946_1122 (size=12154) 2024-11-10T03:16:43,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,002 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111099cd0c1f927d471988d0ffeeb5844ff3_f294ea12149d12df042ace2dfb55c011 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111099cd0c1f927d471988d0ffeeb5844ff3_f294ea12149d12df042ace2dfb55c011 2024-11-10T03:16:44,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/A/1a7935230a7e4260a394104d9b454d4d, store: [table=TestAcidGuarantees family=A region=f294ea12149d12df042ace2dfb55c011] 2024-11-10T03:16:44,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/A/1a7935230a7e4260a394104d9b454d4d is 175, key is test_row_0/A:col10/1731208603092/Put/seqid=0 2024-11-10T03:16:44,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741947_1123 (size=30955) 2024-11-10T03:16:44,011 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=36, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/A/1a7935230a7e4260a394104d9b454d4d 2024-11-10T03:16:44,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/B/0fdf4bd8187246029f31ea7c9beb3d07 is 50, key is test_row_0/B:col10/1731208603092/Put/seqid=0 2024-11-10T03:16:44,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741948_1124 (size=12001) 2024-11-10T03:16:44,068 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/B/0fdf4bd8187246029f31ea7c9beb3d07 2024-11-10T03:16:44,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/C/539a5109cd28463a8b5a86145d6aacfd is 50, key is test_row_0/C:col10/1731208603092/Put/seqid=0 2024-11-10T03:16:44,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741949_1125 (size=12001) 2024-11-10T03:16:44,136 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/C/539a5109cd28463a8b5a86145d6aacfd 2024-11-10T03:16:44,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/A/1a7935230a7e4260a394104d9b454d4d as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/1a7935230a7e4260a394104d9b454d4d 2024-11-10T03:16:44,161 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/1a7935230a7e4260a394104d9b454d4d, entries=150, sequenceid=36, filesize=30.2 K 2024-11-10T03:16:44,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/B/0fdf4bd8187246029f31ea7c9beb3d07 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/0fdf4bd8187246029f31ea7c9beb3d07 2024-11-10T03:16:44,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-10T03:16:44,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,170 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/0fdf4bd8187246029f31ea7c9beb3d07, entries=150, sequenceid=36, filesize=11.7 K 2024-11-10T03:16:44,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/C/539a5109cd28463a8b5a86145d6aacfd as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/539a5109cd28463a8b5a86145d6aacfd 2024-11-10T03:16:44,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,177 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/539a5109cd28463a8b5a86145d6aacfd, entries=150, sequenceid=36, filesize=11.7 K 2024-11-10T03:16:44,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,179 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=0 B/0 for f294ea12149d12df042ace2dfb55c011 in 209ms, sequenceid=36, compaction requested=false 2024-11-10T03:16:44,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2603): Flush status journal for f294ea12149d12df042ace2dfb55c011: 2024-11-10T03:16:44,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:16:44,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=37 2024-11-10T03:16:44,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4169): Remote procedure done, pid=37 2024-11-10T03:16:44,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,184 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-11-10T03:16:44,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,184 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1670 sec 2024-11-10T03:16:44,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,187 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees in 1.1740 sec 2024-11-10T03:16:44,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on f294ea12149d12df042ace2dfb55c011 2024-11-10T03:16:44,362 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f294ea12149d12df042ace2dfb55c011 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-10T03:16:44,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f294ea12149d12df042ace2dfb55c011, store=A 2024-11-10T03:16:44,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:16:44,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f294ea12149d12df042ace2dfb55c011, store=B 2024-11-10T03:16:44,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:16:44,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f294ea12149d12df042ace2dfb55c011, store=C 2024-11-10T03:16:44,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:16:44,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,391 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110a2224e8226e6492e9d3f5bd88434e7cc_f294ea12149d12df042ace2dfb55c011 is 50, key is test_row_0/A:col10/1731208604358/Put/seqid=0 2024-11-10T03:16:44,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:44,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37896 deadline: 1731208664421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:44,434 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731208604400, max timestamp=1731208604429Num uniques 859; 2024-11-10T03:16:44,434 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T03:16:44,437 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731208604400, max timestamp=1731208604429Num uniques 859; 2024-11-10T03:16:44,437 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T03:16:44,440 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731208604400, max timestamp=1731208604429Num uniques 859; 2024-11-10T03:16:44,440 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T03:16:44,441 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:44,441 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:44,441 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:44,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741950_1126 (size=21918) 2024-11-10T03:16:44,451 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:44,458 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110a2224e8226e6492e9d3f5bd88434e7cc_f294ea12149d12df042ace2dfb55c011 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110a2224e8226e6492e9d3f5bd88434e7cc_f294ea12149d12df042ace2dfb55c011 2024-11-10T03:16:44,461 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/A/8a7f7bb8184848e29d35fc61500a2500, store: [table=TestAcidGuarantees family=A region=f294ea12149d12df042ace2dfb55c011] 2024-11-10T03:16:44,462 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/A/8a7f7bb8184848e29d35fc61500a2500 is 175, key is test_row_0/A:col10/1731208604358/Put/seqid=0 2024-11-10T03:16:44,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741951_1127 (size=65323) 2024-11-10T03:16:44,496 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=46, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/A/8a7f7bb8184848e29d35fc61500a2500 2024-11-10T03:16:44,498 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:44,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37948 deadline: 1731208664483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:44,499 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:44,499 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:44,499 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:44,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37958 deadline: 1731208664483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:44,499 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:44,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:44,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:44,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:44,500 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:44,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37914 deadline: 1731208664495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:44,501 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:44,501 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:44,501 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:44,507 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:44,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37968 deadline: 1731208664498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:44,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:44,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:44,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:44,512 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/B/421e2f2d43d04adabd659427d7489107 is 50, key is test_row_0/B:col10/1731208604358/Put/seqid=0 2024-11-10T03:16:44,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741952_1128 (size=12001) 2024-11-10T03:16:44,532 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/B/421e2f2d43d04adabd659427d7489107 2024-11-10T03:16:44,545 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:44,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37896 deadline: 1731208664543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:44,546 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/C/e457e4e05b6848cca6308952496be592 is 50, key is test_row_0/C:col10/1731208604358/Put/seqid=0 2024-11-10T03:16:44,546 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:44,546 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:44,546 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:44,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741953_1129 (size=12001) 2024-11-10T03:16:44,566 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/C/e457e4e05b6848cca6308952496be592 2024-11-10T03:16:44,576 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/A/8a7f7bb8184848e29d35fc61500a2500 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/8a7f7bb8184848e29d35fc61500a2500 2024-11-10T03:16:44,583 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/8a7f7bb8184848e29d35fc61500a2500, entries=350, sequenceid=46, filesize=63.8 K 2024-11-10T03:16:44,585 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/B/421e2f2d43d04adabd659427d7489107 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/421e2f2d43d04adabd659427d7489107 2024-11-10T03:16:44,597 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/421e2f2d43d04adabd659427d7489107, entries=150, sequenceid=46, filesize=11.7 K 2024-11-10T03:16:44,599 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/C/e457e4e05b6848cca6308952496be592 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/e457e4e05b6848cca6308952496be592 2024-11-10T03:16:44,605 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/e457e4e05b6848cca6308952496be592, entries=150, sequenceid=46, filesize=11.7 K 2024-11-10T03:16:44,608 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for f294ea12149d12df042ace2dfb55c011 in 247ms, sequenceid=46, compaction requested=true 2024-11-10T03:16:44,608 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f294ea12149d12df042ace2dfb55c011: 2024-11-10T03:16:44,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f294ea12149d12df042ace2dfb55c011:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T03:16:44,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:16:44,608 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T03:16:44,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f294ea12149d12df042ace2dfb55c011:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T03:16:44,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T03:16:44,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f294ea12149d12df042ace2dfb55c011:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T03:16:44,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-10T03:16:44,609 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T03:16:44,611 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T03:16:44,611 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1541): f294ea12149d12df042ace2dfb55c011/B is initiating minor compaction (all files) 2024-11-10T03:16:44,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on f294ea12149d12df042ace2dfb55c011 2024-11-10T03:16:44,611 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f294ea12149d12df042ace2dfb55c011/B in TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:16:44,611 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/213d7f1b07a94046a71959658c84ed06, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/0fdf4bd8187246029f31ea7c9beb3d07, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/421e2f2d43d04adabd659427d7489107] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp, totalSize=35.2 K 2024-11-10T03:16:44,611 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f294ea12149d12df042ace2dfb55c011 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-10T03:16:44,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f294ea12149d12df042ace2dfb55c011, store=A 2024-11-10T03:16:44,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T03:16:44,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f294ea12149d12df042ace2dfb55c011, store=B 2024-11-10T03:16:44,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T03:16:44,612 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 127233 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T03:16:44,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f294ea12149d12df042ace2dfb55c011, store=C 2024-11-10T03:16:44,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T03:16:44,612 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1541): f294ea12149d12df042ace2dfb55c011/A is initiating minor compaction (all files) 2024-11-10T03:16:44,612 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f294ea12149d12df042ace2dfb55c011/A in TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:16:44,612 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/336f3beea4234c409697878d7c1cf6e0, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/1a7935230a7e4260a394104d9b454d4d, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/8a7f7bb8184848e29d35fc61500a2500] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp, totalSize=124.3 K 2024-11-10T03:16:44,613 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:16:44,613 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. files: [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/336f3beea4234c409697878d7c1cf6e0, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/1a7935230a7e4260a394104d9b454d4d, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/8a7f7bb8184848e29d35fc61500a2500] 2024-11-10T03:16:44,613 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 213d7f1b07a94046a71959658c84ed06, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731208603036 2024-11-10T03:16:44,613 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 336f3beea4234c409697878d7c1cf6e0, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731208603036 2024-11-10T03:16:44,614 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 0fdf4bd8187246029f31ea7c9beb3d07, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1731208603092 2024-11-10T03:16:44,614 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a7935230a7e4260a394104d9b454d4d, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1731208603092 2024-11-10T03:16:44,614 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 421e2f2d43d04adabd659427d7489107, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731208604324 2024-11-10T03:16:44,614 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a7f7bb8184848e29d35fc61500a2500, keycount=350, bloomtype=ROW, size=63.8 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731208604280 2024-11-10T03:16:44,630 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:44,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37948 deadline: 1731208664622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:44,631 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:44,631 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:44,631 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:44,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:44,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37968 deadline: 1731208664626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:44,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:44,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:44,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:44,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:44,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37958 deadline: 1731208664626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:44,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:44,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:44,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:44,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37914 deadline: 1731208664631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:44,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:44,636 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:44,636 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:44,636 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:44,641 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f294ea12149d12df042ace2dfb55c011] 2024-11-10T03:16:44,658 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f294ea12149d12df042ace2dfb55c011#B#compaction#111 average throughput is 0.23 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:16:44,659 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/B/644149eabe39456dadfa1936ebc2f837 is 50, key is test_row_0/B:col10/1731208604358/Put/seqid=0 2024-11-10T03:16:44,665 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110efa89886bf3a4e8fa899569de7d9ec62_f294ea12149d12df042ace2dfb55c011 is 50, key is test_row_0/A:col10/1731208604473/Put/seqid=0 2024-11-10T03:16:44,688 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411103d79491f081e4dec83ad81b526705b63_f294ea12149d12df042ace2dfb55c011 store=[table=TestAcidGuarantees family=A region=f294ea12149d12df042ace2dfb55c011] 2024-11-10T03:16:44,692 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411103d79491f081e4dec83ad81b526705b63_f294ea12149d12df042ace2dfb55c011, store=[table=TestAcidGuarantees family=A region=f294ea12149d12df042ace2dfb55c011] 2024-11-10T03:16:44,692 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411103d79491f081e4dec83ad81b526705b63_f294ea12149d12df042ace2dfb55c011 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f294ea12149d12df042ace2dfb55c011] 2024-11-10T03:16:44,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741954_1130 (size=12104) 2024-11-10T03:16:44,706 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/B/644149eabe39456dadfa1936ebc2f837 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/644149eabe39456dadfa1936ebc2f837 2024-11-10T03:16:44,716 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f294ea12149d12df042ace2dfb55c011/B of f294ea12149d12df042ace2dfb55c011 into 644149eabe39456dadfa1936ebc2f837(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:16:44,716 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f294ea12149d12df042ace2dfb55c011: 2024-11-10T03:16:44,716 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., storeName=f294ea12149d12df042ace2dfb55c011/B, priority=13, startTime=1731208604608; duration=0sec 2024-11-10T03:16:44,716 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T03:16:44,716 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f294ea12149d12df042ace2dfb55c011:B 2024-11-10T03:16:44,716 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T03:16:44,718 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T03:16:44,718 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1541): f294ea12149d12df042ace2dfb55c011/C is initiating minor compaction (all files) 2024-11-10T03:16:44,718 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f294ea12149d12df042ace2dfb55c011/C in TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:16:44,718 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/3d2fc9f3c23846299a1f7aae62b47ae1, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/539a5109cd28463a8b5a86145d6aacfd, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/e457e4e05b6848cca6308952496be592] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp, totalSize=35.2 K 2024-11-10T03:16:44,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741955_1131 (size=14594) 2024-11-10T03:16:44,719 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d2fc9f3c23846299a1f7aae62b47ae1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731208603036 2024-11-10T03:16:44,719 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 539a5109cd28463a8b5a86145d6aacfd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1731208603092 2024-11-10T03:16:44,721 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting e457e4e05b6848cca6308952496be592, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731208604324 2024-11-10T03:16:44,742 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f294ea12149d12df042ace2dfb55c011#C#compaction#114 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:16:44,743 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/C/ab2d92ee0a944503abd0ae5c2b615653 is 50, key is test_row_0/C:col10/1731208604358/Put/seqid=0 2024-11-10T03:16:44,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741956_1132 (size=4469) 2024-11-10T03:16:44,747 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f294ea12149d12df042ace2dfb55c011#A#compaction#112 average throughput is 0.23 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:16:44,748 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/A/8918c6ddff4d44929bc95906305cbbef is 175, key is test_row_0/A:col10/1731208604358/Put/seqid=0 2024-11-10T03:16:44,750 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:44,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37948 deadline: 1731208664743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:44,750 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:44,750 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:44,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:44,751 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:44,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37958 deadline: 1731208664743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:44,751 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:44,751 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:44,751 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:44,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37914 deadline: 1731208664744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:44,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37968 deadline: 1731208664744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:44,751 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:44,751 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:44,752 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:44,752 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:44,752 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:44,754 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:44,754 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:44,754 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:44,757 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:44,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37896 deadline: 1731208664753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:44,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741957_1133 (size=31058) 2024-11-10T03:16:44,758 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:44,758 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:44,758 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:44,766 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/A/8918c6ddff4d44929bc95906305cbbef as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/8918c6ddff4d44929bc95906305cbbef 2024-11-10T03:16:44,775 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f294ea12149d12df042ace2dfb55c011/A of f294ea12149d12df042ace2dfb55c011 into 8918c6ddff4d44929bc95906305cbbef(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:16:44,775 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f294ea12149d12df042ace2dfb55c011: 2024-11-10T03:16:44,775 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., storeName=f294ea12149d12df042ace2dfb55c011/A, priority=13, startTime=1731208604608; duration=0sec 2024-11-10T03:16:44,775 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:16:44,775 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f294ea12149d12df042ace2dfb55c011:A 2024-11-10T03:16:44,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741958_1134 (size=12104) 2024-11-10T03:16:44,797 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/C/ab2d92ee0a944503abd0ae5c2b615653 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/ab2d92ee0a944503abd0ae5c2b615653 2024-11-10T03:16:44,816 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f294ea12149d12df042ace2dfb55c011/C of f294ea12149d12df042ace2dfb55c011 into ab2d92ee0a944503abd0ae5c2b615653(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:16:44,816 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f294ea12149d12df042ace2dfb55c011: 2024-11-10T03:16:44,816 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., storeName=f294ea12149d12df042ace2dfb55c011/C, priority=13, startTime=1731208604608; duration=0sec 2024-11-10T03:16:44,816 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:16:44,816 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f294ea12149d12df042ace2dfb55c011:C 2024-11-10T03:16:44,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:44,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37948 deadline: 1731208664955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:44,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:44,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:44,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:44,968 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:44,968 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:44,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37968 deadline: 1731208664963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:44,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37958 deadline: 1731208664963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:44,969 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:44,969 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:44,969 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:44,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:44,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:44,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:44,976 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:44,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37914 deadline: 1731208664963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:44,978 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:44,978 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:44,978 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:45,064 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:45,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37896 deadline: 1731208665063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:45,065 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:45,065 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:45,065 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:45,121 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,142 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110efa89886bf3a4e8fa899569de7d9ec62_f294ea12149d12df042ace2dfb55c011 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110efa89886bf3a4e8fa899569de7d9ec62_f294ea12149d12df042ace2dfb55c011 2024-11-10T03:16:45,149 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/A/7b5fa90e6cf34eacaed35db4e981d48f, store: [table=TestAcidGuarantees family=A region=f294ea12149d12df042ace2dfb55c011] 2024-11-10T03:16:45,150 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/A/7b5fa90e6cf34eacaed35db4e981d48f is 175, key is test_row_0/A:col10/1731208604473/Put/seqid=0 2024-11-10T03:16:45,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-10T03:16:45,174 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T03:16:45,177 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T03:16:45,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees 2024-11-10T03:16:45,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-10T03:16:45,180 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T03:16:45,182 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T03:16:45,182 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T03:16:45,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741959_1135 (size=39549) 2024-11-10T03:16:45,192 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=83, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/A/7b5fa90e6cf34eacaed35db4e981d48f 2024-11-10T03:16:45,208 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/B/2066322ace80471586cd0f8112f93c4d is 50, key is test_row_0/B:col10/1731208604473/Put/seqid=0 2024-11-10T03:16:45,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741960_1136 (size=12001) 2024-11-10T03:16:45,232 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/B/2066322ace80471586cd0f8112f93c4d 2024-11-10T03:16:45,255 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/C/54bb705b4252426a8ec85fbccbe36d02 is 50, key is test_row_0/C:col10/1731208604473/Put/seqid=0 2024-11-10T03:16:45,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741961_1137 (size=12001) 2024-11-10T03:16:45,278 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/C/54bb705b4252426a8ec85fbccbe36d02 2024-11-10T03:16:45,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:45,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37948 deadline: 1731208665273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:45,281 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:45,281 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:45,282 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:45,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-10T03:16:45,288 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:45,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37958 deadline: 1731208665284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:45,289 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:45,289 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:45,289 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:45,289 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:45,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37914 deadline: 1731208665284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:45,290 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:45,290 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:45,290 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:45,290 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:45,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37968 deadline: 1731208665284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:45,293 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:45,293 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:45,293 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:45,293 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/A/7b5fa90e6cf34eacaed35db4e981d48f as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/7b5fa90e6cf34eacaed35db4e981d48f 2024-11-10T03:16:45,301 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/7b5fa90e6cf34eacaed35db4e981d48f, entries=200, sequenceid=83, filesize=38.6 K 2024-11-10T03:16:45,302 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/B/2066322ace80471586cd0f8112f93c4d as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/2066322ace80471586cd0f8112f93c4d 2024-11-10T03:16:45,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,315 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/2066322ace80471586cd0f8112f93c4d, entries=150, sequenceid=83, filesize=11.7 K 2024-11-10T03:16:45,317 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/C/54bb705b4252426a8ec85fbccbe36d02 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/54bb705b4252426a8ec85fbccbe36d02 2024-11-10T03:16:45,323 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/54bb705b4252426a8ec85fbccbe36d02, entries=150, sequenceid=83, filesize=11.7 K 2024-11-10T03:16:45,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,326 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for f294ea12149d12df042ace2dfb55c011 in 715ms, sequenceid=83, compaction requested=false 2024-11-10T03:16:45,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,326 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f294ea12149d12df042ace2dfb55c011: 2024-11-10T03:16:45,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,335 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-10T03:16:45,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:16:45,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,336 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2902): Flushing f294ea12149d12df042ace2dfb55c011 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-10T03:16:45,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f294ea12149d12df042ace2dfb55c011, store=A 2024-11-10T03:16:45,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:16:45,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f294ea12149d12df042ace2dfb55c011, store=B 2024-11-10T03:16:45,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:16:45,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f294ea12149d12df042ace2dfb55c011, store=C 2024-11-10T03:16:45,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:16:45,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110861a262bf2304e4eb2d06cadf3f73dee_f294ea12149d12df042ace2dfb55c011 is 50, key is test_row_1/A:col10/1731208604622/Put/seqid=0 2024-11-10T03:16:45,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741962_1138 (size=9714) 2024-11-10T03:16:45,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-10T03:16:45,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on f294ea12149d12df042ace2dfb55c011 2024-11-10T03:16:45,576 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. as already flushing 2024-11-10T03:16:45,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,757 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731208605695, max timestamp=1731208605754Num uniques 859; 2024-11-10T03:16:45,757 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T03:16:45,758 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731208605695, max timestamp=1731208605754Num uniques 859; 2024-11-10T03:16:45,758 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T03:16:45,761 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731208605695, max timestamp=1731208605754Num uniques 859; 2024-11-10T03:16:45,761 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T03:16:45,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:45,805 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110861a262bf2304e4eb2d06cadf3f73dee_f294ea12149d12df042ace2dfb55c011 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110861a262bf2304e4eb2d06cadf3f73dee_f294ea12149d12df042ace2dfb55c011 2024-11-10T03:16:45,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/A/6ecb82eaee404670b0fea6a033ab7d3f, store: [table=TestAcidGuarantees family=A region=f294ea12149d12df042ace2dfb55c011] 2024-11-10T03:16:45,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-10T03:16:45,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/A/6ecb82eaee404670b0fea6a033ab7d3f is 175, key is test_row_1/A:col10/1731208604622/Put/seqid=0 2024-11-10T03:16:45,820 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:45,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37958 deadline: 1731208665812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:45,821 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:45,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37948 deadline: 1731208665814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:45,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:45,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:45,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:45,821 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:45,821 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:45,821 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:45,821 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:45,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37914 deadline: 1731208665815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:45,822 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:45,822 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:45,822 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:45,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:45,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37896 deadline: 1731208665816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:45,822 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:45,822 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:45,822 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:45,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741963_1139 (size=22361) 2024-11-10T03:16:45,826 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=95, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/A/6ecb82eaee404670b0fea6a033ab7d3f 2024-11-10T03:16:45,826 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:45,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37968 deadline: 1731208665821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:45,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:45,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:45,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:45,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/B/71a175df075c424fa4d52439da8eb5fc is 50, key is test_row_1/B:col10/1731208604622/Put/seqid=0 2024-11-10T03:16:45,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741964_1140 (size=9657) 2024-11-10T03:16:45,934 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:45,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37948 deadline: 1731208665933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:45,937 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:45,937 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:45,937 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:45,937 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:45,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37896 deadline: 1731208665934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:45,938 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:45,938 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:45,938 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:45,938 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:45,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37968 deadline: 1731208665934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:45,938 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:45,938 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:45,938 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:45,939 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:45,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37958 deadline: 1731208665934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:45,939 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:45,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:45,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:45,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37914 deadline: 1731208665935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:45,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:45,940 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:45,940 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:45,940 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:46,149 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:46,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37968 deadline: 1731208666144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:46,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:46,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37958 deadline: 1731208666144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:46,150 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:46,150 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:46,150 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:46,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:46,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:46,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:46,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:46,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37948 deadline: 1731208666144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:46,150 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:46,151 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:46,151 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:46,151 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:46,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37896 deadline: 1731208666144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:46,151 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:46,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37914 deadline: 1731208666144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:46,151 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:46,151 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:46,151 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:46,152 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:46,152 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:46,152 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:46,229 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-10T03:16:46,261 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/B/71a175df075c424fa4d52439da8eb5fc 2024-11-10T03:16:46,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/C/1084770c5a5e4b54ba8de4f63e5b7ada is 50, key is test_row_1/C:col10/1731208604622/Put/seqid=0 2024-11-10T03:16:46,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-10T03:16:46,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741965_1141 (size=9657) 2024-11-10T03:16:46,323 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/C/1084770c5a5e4b54ba8de4f63e5b7ada 2024-11-10T03:16:46,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/A/6ecb82eaee404670b0fea6a033ab7d3f as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/6ecb82eaee404670b0fea6a033ab7d3f 2024-11-10T03:16:46,340 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/6ecb82eaee404670b0fea6a033ab7d3f, entries=100, sequenceid=95, filesize=21.8 K 2024-11-10T03:16:46,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/B/71a175df075c424fa4d52439da8eb5fc as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/71a175df075c424fa4d52439da8eb5fc 2024-11-10T03:16:46,369 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/71a175df075c424fa4d52439da8eb5fc, entries=100, sequenceid=95, filesize=9.4 K 2024-11-10T03:16:46,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/C/1084770c5a5e4b54ba8de4f63e5b7ada as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/1084770c5a5e4b54ba8de4f63e5b7ada 2024-11-10T03:16:46,383 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/1084770c5a5e4b54ba8de4f63e5b7ada, entries=100, sequenceid=95, filesize=9.4 K 2024-11-10T03:16:46,390 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for f294ea12149d12df042ace2dfb55c011 in 1053ms, sequenceid=95, compaction requested=true 2024-11-10T03:16:46,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2603): Flush status journal for f294ea12149d12df042ace2dfb55c011: 2024-11-10T03:16:46,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:16:46,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=39 2024-11-10T03:16:46,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4169): Remote procedure done, pid=39 2024-11-10T03:16:46,402 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-10T03:16:46,402 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2150 sec 2024-11-10T03:16:46,408 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees in 1.2290 sec 2024-11-10T03:16:46,464 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f294ea12149d12df042ace2dfb55c011 3/3 column families, dataSize=154.31 KB heapSize=440.77 KB 2024-11-10T03:16:46,464 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f294ea12149d12df042ace2dfb55c011, store=A 2024-11-10T03:16:46,465 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-10T03:16:46,465 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f294ea12149d12df042ace2dfb55c011, store=B 2024-11-10T03:16:46,465 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-10T03:16:46,465 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f294ea12149d12df042ace2dfb55c011, store=C 2024-11-10T03:16:46,465 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-10T03:16:46,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on f294ea12149d12df042ace2dfb55c011 2024-11-10T03:16:46,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:46,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37896 deadline: 1731208666468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:46,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:46,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37958 deadline: 1731208666468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:46,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:46,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37948 deadline: 1731208666475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:46,485 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:46,485 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:46,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37914 deadline: 1731208666477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:46,485 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:46,485 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:46,485 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:46,486 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:46,486 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:46,486 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:46,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37968 deadline: 1731208666479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:46,486 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:46,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:46,486 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:46,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:46,486 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:46,487 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:46,487 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:46,487 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:46,487 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:46,499 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110c2d0c77192d846aeb94d3a3fe5861782_f294ea12149d12df042ace2dfb55c011 is 50, key is test_row_0/A:col10/1731208605818/Put/seqid=0 2024-11-10T03:16:46,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741966_1142 (size=12304) 2024-11-10T03:16:46,531 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:46,540 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110c2d0c77192d846aeb94d3a3fe5861782_f294ea12149d12df042ace2dfb55c011 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110c2d0c77192d846aeb94d3a3fe5861782_f294ea12149d12df042ace2dfb55c011 2024-11-10T03:16:46,541 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/A/ff659bfa7d7e445db204649d842022a2, store: [table=TestAcidGuarantees family=A region=f294ea12149d12df042ace2dfb55c011] 2024-11-10T03:16:46,542 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/A/ff659bfa7d7e445db204649d842022a2 is 175, key is test_row_0/A:col10/1731208605818/Put/seqid=0 2024-11-10T03:16:46,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741967_1143 (size=31105) 2024-11-10T03:16:46,562 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=134, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/A/ff659bfa7d7e445db204649d842022a2 2024-11-10T03:16:46,583 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/B/be3e0aa1bcfd4b1db9280d355a709946 is 50, key is test_row_0/B:col10/1731208605818/Put/seqid=0 2024-11-10T03:16:46,598 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:46,598 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:46,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37896 deadline: 1731208666593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:46,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37958 deadline: 1731208666594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:46,598 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:46,598 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:46,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37948 deadline: 1731208666594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:46,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37968 deadline: 1731208666594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:46,599 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:46,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37914 deadline: 1731208666594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:46,599 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:46,599 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:46,599 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:46,599 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:46,599 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:46,599 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:46,600 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:46,600 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:46,600 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:46,600 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:46,601 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:46,601 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:46,601 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:46,601 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:46,601 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:46,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741968_1144 (size=12151) 2024-11-10T03:16:46,614 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/B/be3e0aa1bcfd4b1db9280d355a709946 2024-11-10T03:16:46,639 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/C/da0d3cb699e3418abcb877159c8fc1d4 is 50, key is test_row_0/C:col10/1731208605818/Put/seqid=0 2024-11-10T03:16:46,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741969_1145 (size=12151) 2024-11-10T03:16:46,656 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/C/da0d3cb699e3418abcb877159c8fc1d4 2024-11-10T03:16:46,663 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/A/ff659bfa7d7e445db204649d842022a2 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/ff659bfa7d7e445db204649d842022a2 2024-11-10T03:16:46,669 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/ff659bfa7d7e445db204649d842022a2, entries=150, sequenceid=134, filesize=30.4 K 2024-11-10T03:16:46,672 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/B/be3e0aa1bcfd4b1db9280d355a709946 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/be3e0aa1bcfd4b1db9280d355a709946 2024-11-10T03:16:46,678 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/be3e0aa1bcfd4b1db9280d355a709946, entries=150, sequenceid=134, filesize=11.9 K 2024-11-10T03:16:46,679 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/C/da0d3cb699e3418abcb877159c8fc1d4 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/da0d3cb699e3418abcb877159c8fc1d4 2024-11-10T03:16:46,684 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/da0d3cb699e3418abcb877159c8fc1d4, entries=150, sequenceid=134, filesize=11.9 K 2024-11-10T03:16:46,685 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~440.72 KB/451296, currentSize=26.84 KB/27480 for f294ea12149d12df042ace2dfb55c011 in 221ms, sequenceid=134, compaction requested=true 2024-11-10T03:16:46,685 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f294ea12149d12df042ace2dfb55c011: 2024-11-10T03:16:46,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f294ea12149d12df042ace2dfb55c011:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T03:16:46,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:16:46,685 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T03:16:46,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f294ea12149d12df042ace2dfb55c011:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T03:16:46,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T03:16:46,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f294ea12149d12df042ace2dfb55c011:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T03:16:46,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-10T03:16:46,685 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T03:16:46,687 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45913 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T03:16:46,687 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1541): f294ea12149d12df042ace2dfb55c011/B is initiating minor compaction (all files) 2024-11-10T03:16:46,687 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f294ea12149d12df042ace2dfb55c011/B in TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:16:46,687 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/644149eabe39456dadfa1936ebc2f837, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/2066322ace80471586cd0f8112f93c4d, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/71a175df075c424fa4d52439da8eb5fc, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/be3e0aa1bcfd4b1db9280d355a709946] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp, totalSize=44.8 K 2024-11-10T03:16:46,688 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 124073 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T03:16:46,688 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 644149eabe39456dadfa1936ebc2f837, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731208604324 2024-11-10T03:16:46,688 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1541): f294ea12149d12df042ace2dfb55c011/A is initiating minor compaction (all files) 2024-11-10T03:16:46,688 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f294ea12149d12df042ace2dfb55c011/A in TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:16:46,688 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/8918c6ddff4d44929bc95906305cbbef, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/7b5fa90e6cf34eacaed35db4e981d48f, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/6ecb82eaee404670b0fea6a033ab7d3f, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/ff659bfa7d7e445db204649d842022a2] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp, totalSize=121.2 K 2024-11-10T03:16:46,688 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 2066322ace80471586cd0f8112f93c4d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731208604473 2024-11-10T03:16:46,688 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:16:46,688 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. files: [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/8918c6ddff4d44929bc95906305cbbef, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/7b5fa90e6cf34eacaed35db4e981d48f, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/6ecb82eaee404670b0fea6a033ab7d3f, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/ff659bfa7d7e445db204649d842022a2] 2024-11-10T03:16:46,688 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 71a175df075c424fa4d52439da8eb5fc, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731208604622 2024-11-10T03:16:46,689 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting be3e0aa1bcfd4b1db9280d355a709946, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731208605815 2024-11-10T03:16:46,689 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8918c6ddff4d44929bc95906305cbbef, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731208604324 2024-11-10T03:16:46,691 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b5fa90e6cf34eacaed35db4e981d48f, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731208604430 2024-11-10T03:16:46,692 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ecb82eaee404670b0fea6a033ab7d3f, keycount=100, bloomtype=ROW, size=21.8 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731208604622 2024-11-10T03:16:46,693 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting ff659bfa7d7e445db204649d842022a2, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731208605815 2024-11-10T03:16:46,714 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f294ea12149d12df042ace2dfb55c011#B#compaction#123 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:16:46,714 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/B/a92fbcaade63481392676c6bbb53a0d6 is 50, key is test_row_0/B:col10/1731208605818/Put/seqid=0 2024-11-10T03:16:46,729 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f294ea12149d12df042ace2dfb55c011] 2024-11-10T03:16:46,749 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411106996a589f6fc417081e5a65e5608181a_f294ea12149d12df042ace2dfb55c011 store=[table=TestAcidGuarantees family=A region=f294ea12149d12df042ace2dfb55c011] 2024-11-10T03:16:46,752 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411106996a589f6fc417081e5a65e5608181a_f294ea12149d12df042ace2dfb55c011, store=[table=TestAcidGuarantees family=A region=f294ea12149d12df042ace2dfb55c011] 2024-11-10T03:16:46,752 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411106996a589f6fc417081e5a65e5608181a_f294ea12149d12df042ace2dfb55c011 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f294ea12149d12df042ace2dfb55c011] 2024-11-10T03:16:46,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741970_1146 (size=12391) 2024-11-10T03:16:46,790 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/B/a92fbcaade63481392676c6bbb53a0d6 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/a92fbcaade63481392676c6bbb53a0d6 2024-11-10T03:16:46,800 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f294ea12149d12df042ace2dfb55c011/B of f294ea12149d12df042ace2dfb55c011 into a92fbcaade63481392676c6bbb53a0d6(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:16:46,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741971_1147 (size=4469) 2024-11-10T03:16:46,800 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f294ea12149d12df042ace2dfb55c011: 2024-11-10T03:16:46,800 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., storeName=f294ea12149d12df042ace2dfb55c011/B, priority=12, startTime=1731208606685; duration=0sec 2024-11-10T03:16:46,800 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T03:16:46,801 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f294ea12149d12df042ace2dfb55c011:B 2024-11-10T03:16:46,801 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T03:16:46,804 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f294ea12149d12df042ace2dfb55c011#A#compaction#124 average throughput is 0.33 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:16:46,805 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/A/d94adcb6627d447399754c787e225d93 is 175, key is test_row_0/A:col10/1731208605818/Put/seqid=0 2024-11-10T03:16:46,807 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45913 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T03:16:46,807 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1541): f294ea12149d12df042ace2dfb55c011/C is initiating minor compaction (all files) 2024-11-10T03:16:46,807 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f294ea12149d12df042ace2dfb55c011/C in TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:16:46,808 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/ab2d92ee0a944503abd0ae5c2b615653, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/54bb705b4252426a8ec85fbccbe36d02, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/1084770c5a5e4b54ba8de4f63e5b7ada, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/da0d3cb699e3418abcb877159c8fc1d4] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp, totalSize=44.8 K 2024-11-10T03:16:46,808 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting ab2d92ee0a944503abd0ae5c2b615653, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731208604324 2024-11-10T03:16:46,809 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 54bb705b4252426a8ec85fbccbe36d02, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731208604473 2024-11-10T03:16:46,810 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 1084770c5a5e4b54ba8de4f63e5b7ada, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731208604622 2024-11-10T03:16:46,810 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting da0d3cb699e3418abcb877159c8fc1d4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731208605815 2024-11-10T03:16:46,810 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f294ea12149d12df042ace2dfb55c011 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-10T03:16:46,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f294ea12149d12df042ace2dfb55c011, store=A 2024-11-10T03:16:46,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:16:46,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f294ea12149d12df042ace2dfb55c011, store=B 2024-11-10T03:16:46,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:16:46,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f294ea12149d12df042ace2dfb55c011, store=C 2024-11-10T03:16:46,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:16:46,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on f294ea12149d12df042ace2dfb55c011 2024-11-10T03:16:46,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741972_1148 (size=31345) 2024-11-10T03:16:46,852 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/A/d94adcb6627d447399754c787e225d93 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/d94adcb6627d447399754c787e225d93 2024-11-10T03:16:46,855 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f294ea12149d12df042ace2dfb55c011#C#compaction#125 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:16:46,856 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/C/3aca2c39adad4adea221f8c80be0c51e is 50, key is test_row_0/C:col10/1731208605818/Put/seqid=0 2024-11-10T03:16:46,864 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:46,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37896 deadline: 1731208666854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:46,865 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f294ea12149d12df042ace2dfb55c011/A of f294ea12149d12df042ace2dfb55c011 into d94adcb6627d447399754c787e225d93(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:16:46,865 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:46,865 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f294ea12149d12df042ace2dfb55c011: 2024-11-10T03:16:46,865 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:46,865 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., storeName=f294ea12149d12df042ace2dfb55c011/A, priority=12, startTime=1731208606685; duration=0sec 2024-11-10T03:16:46,865 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:46,865 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:16:46,865 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f294ea12149d12df042ace2dfb55c011:A 2024-11-10T03:16:46,866 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:46,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37958 deadline: 1731208666859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:46,866 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110adb34b7b49354c09b5b8a00ee4285801_f294ea12149d12df042ace2dfb55c011 is 50, key is test_row_0/A:col10/1731208606809/Put/seqid=0 2024-11-10T03:16:46,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:46,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:46,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:46,867 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:46,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37914 deadline: 1731208666860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:46,867 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:46,867 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:46,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37968 deadline: 1731208666861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:46,868 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:46,868 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:46,868 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:46,868 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:46,869 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:46,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:46,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37948 deadline: 1731208666864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:46,876 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:46,876 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:46,876 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:46,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741974_1150 (size=12304) 2024-11-10T03:16:46,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741973_1149 (size=12391) 2024-11-10T03:16:46,975 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:46,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37896 deadline: 1731208666974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:46,976 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:46,976 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:46,976 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:46,976 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:46,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37968 deadline: 1731208666974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:46,977 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:46,977 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:46,977 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:46,978 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:46,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37958 deadline: 1731208666974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:46,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:46,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:46,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:46,982 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:46,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37914 deadline: 1731208666981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:46,983 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:46,983 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:46,983 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:46,991 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:46,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37948 deadline: 1731208666983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:46,992 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:46,992 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:46,993 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:47,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:47,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37896 deadline: 1731208667184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:47,195 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:47,195 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:47,195 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:47,196 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:47,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37968 deadline: 1731208667186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:47,198 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:47,198 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:47,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:47,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:47,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:47,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37958 deadline: 1731208667186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:47,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:47,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:47,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:47,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37914 deadline: 1731208667193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:47,205 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:47,205 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:47,205 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:47,210 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:47,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37948 deadline: 1731208667204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:47,216 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:47,216 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:47,216 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:47,309 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:47,318 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110adb34b7b49354c09b5b8a00ee4285801_f294ea12149d12df042ace2dfb55c011 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110adb34b7b49354c09b5b8a00ee4285801_f294ea12149d12df042ace2dfb55c011 2024-11-10T03:16:47,320 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/A/c29ea343310b4db3801e38b22c82f03c, store: [table=TestAcidGuarantees family=A region=f294ea12149d12df042ace2dfb55c011] 2024-11-10T03:16:47,321 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/A/c29ea343310b4db3801e38b22c82f03c is 175, key is test_row_0/A:col10/1731208606809/Put/seqid=0 2024-11-10T03:16:47,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-10T03:16:47,324 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T03:16:47,329 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T03:16:47,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-10T03:16:47,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-10T03:16:47,331 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T03:16:47,332 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T03:16:47,332 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T03:16:47,333 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/C/3aca2c39adad4adea221f8c80be0c51e as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/3aca2c39adad4adea221f8c80be0c51e 2024-11-10T03:16:47,347 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f294ea12149d12df042ace2dfb55c011/C of f294ea12149d12df042ace2dfb55c011 into 3aca2c39adad4adea221f8c80be0c51e(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:16:47,347 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f294ea12149d12df042ace2dfb55c011: 2024-11-10T03:16:47,348 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., storeName=f294ea12149d12df042ace2dfb55c011/C, priority=12, startTime=1731208606685; duration=0sec 2024-11-10T03:16:47,348 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:16:47,348 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f294ea12149d12df042ace2dfb55c011:C 2024-11-10T03:16:47,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741975_1151 (size=31105) 2024-11-10T03:16:47,352 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=146, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/A/c29ea343310b4db3801e38b22c82f03c 2024-11-10T03:16:47,375 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/B/7d5e517c23014f818f98108d8254a81d is 50, key is test_row_0/B:col10/1731208606809/Put/seqid=0 2024-11-10T03:16:47,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741976_1152 (size=12151) 2024-11-10T03:16:47,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-10T03:16:47,490 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-10T03:16:47,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:16:47,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. as already flushing 2024-11-10T03:16:47,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:16:47,493 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:47,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:47,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:47,507 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:47,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37968 deadline: 1731208667504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:47,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:47,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:47,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:47,509 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:47,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37896 deadline: 1731208667505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:47,509 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:47,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37958 deadline: 1731208667506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:47,512 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:47,512 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:47,512 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:47,513 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:47,513 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:47,513 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:47,520 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:47,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37914 deadline: 1731208667520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:47,523 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:47,523 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:47,523 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:47,526 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:47,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37948 deadline: 1731208667525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:47,527 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:47,527 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:47,527 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:47,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-10T03:16:47,652 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-10T03:16:47,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:16:47,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. as already flushing 2024-11-10T03:16:47,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:16:47,656 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:47,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:47,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:47,803 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=146 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/B/7d5e517c23014f818f98108d8254a81d 2024-11-10T03:16:47,810 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-10T03:16:47,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:16:47,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. as already flushing 2024-11-10T03:16:47,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:16:47,811 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:47,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:47,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:47,826 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/C/93e15adcf384445b99e700001c6d0776 is 50, key is test_row_0/C:col10/1731208606809/Put/seqid=0 2024-11-10T03:16:47,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741977_1153 (size=12151) 2024-11-10T03:16:47,843 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=146 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/C/93e15adcf384445b99e700001c6d0776 2024-11-10T03:16:47,862 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/A/c29ea343310b4db3801e38b22c82f03c as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/c29ea343310b4db3801e38b22c82f03c 2024-11-10T03:16:47,878 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/c29ea343310b4db3801e38b22c82f03c, entries=150, sequenceid=146, filesize=30.4 K 2024-11-10T03:16:47,880 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/B/7d5e517c23014f818f98108d8254a81d as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/7d5e517c23014f818f98108d8254a81d 2024-11-10T03:16:47,894 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/7d5e517c23014f818f98108d8254a81d, entries=150, sequenceid=146, filesize=11.9 K 2024-11-10T03:16:47,896 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/C/93e15adcf384445b99e700001c6d0776 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/93e15adcf384445b99e700001c6d0776 2024-11-10T03:16:47,909 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/93e15adcf384445b99e700001c6d0776, entries=150, sequenceid=146, filesize=11.9 K 2024-11-10T03:16:47,916 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=120.76 KB/123660 for f294ea12149d12df042ace2dfb55c011 in 1106ms, sequenceid=146, compaction requested=false 2024-11-10T03:16:47,916 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f294ea12149d12df042ace2dfb55c011: 2024-11-10T03:16:47,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-10T03:16:47,964 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-10T03:16:47,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:16:47,965 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing f294ea12149d12df042ace2dfb55c011 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-10T03:16:47,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f294ea12149d12df042ace2dfb55c011, store=A 2024-11-10T03:16:47,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T03:16:47,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f294ea12149d12df042ace2dfb55c011, store=B 2024-11-10T03:16:47,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T03:16:47,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f294ea12149d12df042ace2dfb55c011, store=C 2024-11-10T03:16:47,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T03:16:47,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411106e9af77f38934557b860b6fc2dd67835_f294ea12149d12df042ace2dfb55c011 is 50, key is test_row_0/A:col10/1731208606854/Put/seqid=0 2024-11-10T03:16:47,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741978_1154 (size=12304) 2024-11-10T03:16:48,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on f294ea12149d12df042ace2dfb55c011 2024-11-10T03:16:48,017 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. as already flushing 2024-11-10T03:16:48,036 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:48,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37968 deadline: 1731208668031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:48,037 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:48,038 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:48,038 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:48,039 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:48,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37958 deadline: 1731208668033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:48,039 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:48,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37948 deadline: 1731208668034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:48,039 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:48,039 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:48,040 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:48,040 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:48,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37914 deadline: 1731208668034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:48,041 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:48,042 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:48,042 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:48,043 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:48,043 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:48,043 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:48,044 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:48,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37896 deadline: 1731208668038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:48,044 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:48,044 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:48,044 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:48,144 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:48,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37968 deadline: 1731208668143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:48,145 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:48,146 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:48,146 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:48,146 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:48,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37958 deadline: 1731208668143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:48,147 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:48,147 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:48,147 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:48,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:48,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37896 deadline: 1731208668153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:48,159 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:48,159 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:48,159 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:48,354 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:48,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37958 deadline: 1731208668353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:48,355 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:48,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37968 deadline: 1731208668353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:48,355 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:48,355 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:48,355 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:48,356 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:48,356 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:48,356 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:48,366 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:48,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37896 deadline: 1731208668363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:48,367 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:48,367 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:48,367 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:48,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:48,394 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411106e9af77f38934557b860b6fc2dd67835_f294ea12149d12df042ace2dfb55c011 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411106e9af77f38934557b860b6fc2dd67835_f294ea12149d12df042ace2dfb55c011 2024-11-10T03:16:48,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/A/cad986c7afc645a5b578e01c9bf4ab7b, store: [table=TestAcidGuarantees family=A region=f294ea12149d12df042ace2dfb55c011] 2024-11-10T03:16:48,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/A/cad986c7afc645a5b578e01c9bf4ab7b is 175, key is test_row_0/A:col10/1731208606854/Put/seqid=0 2024-11-10T03:16:48,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741979_1155 (size=31105) 2024-11-10T03:16:48,411 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=169, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/A/cad986c7afc645a5b578e01c9bf4ab7b 2024-11-10T03:16:48,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/B/31fb514dd41a43c58ce6eec10b0d08d0 is 50, key is test_row_0/B:col10/1731208606854/Put/seqid=0 2024-11-10T03:16:48,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741980_1156 (size=12151) 2024-11-10T03:16:48,439 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/B/31fb514dd41a43c58ce6eec10b0d08d0 2024-11-10T03:16:48,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-10T03:16:48,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/C/0deab162ce8d4b3ebbcf348ad6dac0a6 is 50, key is test_row_0/C:col10/1731208606854/Put/seqid=0 2024-11-10T03:16:48,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741981_1157 (size=12151) 2024-11-10T03:16:48,665 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:48,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37958 deadline: 1731208668665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:48,666 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:48,666 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:48,666 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:48,666 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:48,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37968 deadline: 1731208668665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:48,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:48,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:48,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:48,676 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:48,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37896 deadline: 1731208668674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:48,677 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:48,677 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:48,677 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:48,898 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/C/0deab162ce8d4b3ebbcf348ad6dac0a6 2024-11-10T03:16:48,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/A/cad986c7afc645a5b578e01c9bf4ab7b as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/cad986c7afc645a5b578e01c9bf4ab7b 2024-11-10T03:16:48,913 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/cad986c7afc645a5b578e01c9bf4ab7b, entries=150, sequenceid=169, filesize=30.4 K 2024-11-10T03:16:48,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/B/31fb514dd41a43c58ce6eec10b0d08d0 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/31fb514dd41a43c58ce6eec10b0d08d0 2024-11-10T03:16:48,923 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/31fb514dd41a43c58ce6eec10b0d08d0, entries=150, sequenceid=169, filesize=11.9 K 2024-11-10T03:16:48,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/C/0deab162ce8d4b3ebbcf348ad6dac0a6 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/0deab162ce8d4b3ebbcf348ad6dac0a6 2024-11-10T03:16:48,935 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/0deab162ce8d4b3ebbcf348ad6dac0a6, entries=150, sequenceid=169, filesize=11.9 K 2024-11-10T03:16:48,936 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for f294ea12149d12df042ace2dfb55c011 in 971ms, sequenceid=169, compaction requested=true 2024-11-10T03:16:48,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for f294ea12149d12df042ace2dfb55c011: 2024-11-10T03:16:48,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:16:48,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-10T03:16:48,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-10T03:16:48,943 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-10T03:16:48,943 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6090 sec 2024-11-10T03:16:48,945 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 1.6150 sec 2024-11-10T03:16:49,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on f294ea12149d12df042ace2dfb55c011 2024-11-10T03:16:49,060 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f294ea12149d12df042ace2dfb55c011 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-10T03:16:49,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f294ea12149d12df042ace2dfb55c011, store=A 2024-11-10T03:16:49,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:16:49,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f294ea12149d12df042ace2dfb55c011, store=B 2024-11-10T03:16:49,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:16:49,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f294ea12149d12df042ace2dfb55c011, store=C 2024-11-10T03:16:49,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:16:49,071 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111032b14e7cd820457a9277167dabcd1407_f294ea12149d12df042ace2dfb55c011 is 50, key is test_row_0/A:col10/1731208608034/Put/seqid=0 2024-11-10T03:16:49,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741982_1158 (size=17284) 2024-11-10T03:16:49,092 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:49,097 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111032b14e7cd820457a9277167dabcd1407_f294ea12149d12df042ace2dfb55c011 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111032b14e7cd820457a9277167dabcd1407_f294ea12149d12df042ace2dfb55c011 2024-11-10T03:16:49,100 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/A/8045e9cb51c64fe9bc686c5f7bca09a8, store: [table=TestAcidGuarantees family=A region=f294ea12149d12df042ace2dfb55c011] 2024-11-10T03:16:49,101 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/A/8045e9cb51c64fe9bc686c5f7bca09a8 is 175, key is test_row_0/A:col10/1731208608034/Put/seqid=0 2024-11-10T03:16:49,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741983_1159 (size=48389) 2024-11-10T03:16:49,128 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:49,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37948 deadline: 1731208669122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:49,129 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:49,129 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:49,129 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:49,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:49,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37914 deadline: 1731208669129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:49,136 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:49,136 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:49,136 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:49,174 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:49,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37968 deadline: 1731208669173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:49,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:49,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:49,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:49,175 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:49,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37958 deadline: 1731208669173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:49,176 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:49,176 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:49,176 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:49,193 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:49,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37896 deadline: 1731208669183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:49,196 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:49,196 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:49,197 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:49,238 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:49,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37948 deadline: 1731208669233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:49,240 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:49,240 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:49,240 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:49,249 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:49,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37914 deadline: 1731208669248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:49,250 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:49,250 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:49,250 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:49,467 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:49,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37948 deadline: 1731208669457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:49,467 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:49,467 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:49,467 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:49,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:49,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37914 deadline: 1731208669457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:49,471 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:49,471 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:49,471 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:49,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-10T03:16:49,473 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T03:16:49,474 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T03:16:49,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-10T03:16:49,480 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T03:16:49,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-10T03:16:49,481 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T03:16:49,482 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T03:16:49,511 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=182, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/A/8045e9cb51c64fe9bc686c5f7bca09a8 2024-11-10T03:16:49,539 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/B/0aa5edd7d5024eb9866743af026efd58 is 50, key is test_row_0/B:col10/1731208608034/Put/seqid=0 2024-11-10T03:16:49,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741984_1160 (size=12151) 2024-11-10T03:16:49,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-10T03:16:49,655 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-10T03:16:49,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:16:49,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. as already flushing 2024-11-10T03:16:49,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:16:49,655 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:49,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:49,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:49,786 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:49,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37948 deadline: 1731208669775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:49,786 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:49,787 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:49,787 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:49,787 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:49,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37914 deadline: 1731208669783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:49,788 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:49,788 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:49,788 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:49,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-10T03:16:49,813 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-10T03:16:49,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:16:49,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. as already flushing 2024-11-10T03:16:49,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:16:49,814 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:49,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:49,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:49,967 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-10T03:16:49,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:16:49,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. as already flushing 2024-11-10T03:16:49,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:16:49,968 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:49,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:49,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:49,984 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/B/0aa5edd7d5024eb9866743af026efd58 2024-11-10T03:16:49,994 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/C/5383bb8043d8467fa677ef5c252776c7 is 50, key is test_row_0/C:col10/1731208608034/Put/seqid=0 2024-11-10T03:16:50,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741985_1161 (size=12151) 2024-11-10T03:16:50,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-10T03:16:50,122 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-10T03:16:50,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:16:50,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. as already flushing 2024-11-10T03:16:50,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:16:50,122 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:50,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:50,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:50,196 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:50,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37958 deadline: 1731208670189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:50,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:50,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37968 deadline: 1731208670197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:50,197 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 394d2222f724,38139,1731208572054 for 'test_row_1' in f294ea12149d12df042ace2dfb55c011 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T03:16:50,198 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:50,198 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:50,198 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:50,201 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 394d2222f724,38139,1731208572054 for 'test_row_2' in f294ea12149d12df042ace2dfb55c011 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T03:16:50,202 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:50,202 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:50,202 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:50,217 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:50,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37896 deadline: 1731208670213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:50,219 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 394d2222f724,38139,1731208572054 for 'test_row_1' in f294ea12149d12df042ace2dfb55c011 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T03:16:50,220 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:50,220 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:50,220 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:50,276 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-10T03:16:50,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:16:50,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. as already flushing 2024-11-10T03:16:50,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:16:50,276 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:50,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:50,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:50,303 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:50,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37948 deadline: 1731208670293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:50,304 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:50,304 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:50,304 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:50,307 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:50,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37914 deadline: 1731208670303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:50,308 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:50,308 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:50,308 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:50,432 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-10T03:16:50,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:16:50,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. as already flushing 2024-11-10T03:16:50,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:16:50,433 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:50,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:50,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:50,474 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/C/5383bb8043d8467fa677ef5c252776c7 2024-11-10T03:16:50,485 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/A/8045e9cb51c64fe9bc686c5f7bca09a8 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/8045e9cb51c64fe9bc686c5f7bca09a8 2024-11-10T03:16:50,495 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/8045e9cb51c64fe9bc686c5f7bca09a8, entries=250, sequenceid=182, filesize=47.3 K 2024-11-10T03:16:50,496 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/B/0aa5edd7d5024eb9866743af026efd58 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/0aa5edd7d5024eb9866743af026efd58 2024-11-10T03:16:50,504 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/0aa5edd7d5024eb9866743af026efd58, entries=150, sequenceid=182, filesize=11.9 K 2024-11-10T03:16:50,506 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/C/5383bb8043d8467fa677ef5c252776c7 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/5383bb8043d8467fa677ef5c252776c7 2024-11-10T03:16:50,512 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/5383bb8043d8467fa677ef5c252776c7, entries=150, sequenceid=182, filesize=11.9 K 2024-11-10T03:16:50,513 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for f294ea12149d12df042ace2dfb55c011 in 1455ms, sequenceid=182, compaction requested=true 2024-11-10T03:16:50,513 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f294ea12149d12df042ace2dfb55c011: 2024-11-10T03:16:50,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f294ea12149d12df042ace2dfb55c011:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T03:16:50,514 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T03:16:50,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:16:50,514 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T03:16:50,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f294ea12149d12df042ace2dfb55c011:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T03:16:50,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:16:50,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f294ea12149d12df042ace2dfb55c011:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T03:16:50,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T03:16:50,515 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141944 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T03:16:50,515 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1541): f294ea12149d12df042ace2dfb55c011/A is initiating minor compaction (all files) 2024-11-10T03:16:50,515 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f294ea12149d12df042ace2dfb55c011/A in TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:16:50,516 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/d94adcb6627d447399754c787e225d93, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/c29ea343310b4db3801e38b22c82f03c, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/cad986c7afc645a5b578e01c9bf4ab7b, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/8045e9cb51c64fe9bc686c5f7bca09a8] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp, totalSize=138.6 K 2024-11-10T03:16:50,516 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:16:50,516 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. files: [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/d94adcb6627d447399754c787e225d93, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/c29ea343310b4db3801e38b22c82f03c, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/cad986c7afc645a5b578e01c9bf4ab7b, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/8045e9cb51c64fe9bc686c5f7bca09a8] 2024-11-10T03:16:50,517 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48844 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T03:16:50,517 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting d94adcb6627d447399754c787e225d93, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731208605815 2024-11-10T03:16:50,517 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1541): f294ea12149d12df042ace2dfb55c011/B is initiating minor compaction (all files) 2024-11-10T03:16:50,517 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f294ea12149d12df042ace2dfb55c011/B in TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:16:50,517 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/a92fbcaade63481392676c6bbb53a0d6, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/7d5e517c23014f818f98108d8254a81d, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/31fb514dd41a43c58ce6eec10b0d08d0, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/0aa5edd7d5024eb9866743af026efd58] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp, totalSize=47.7 K 2024-11-10T03:16:50,518 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting a92fbcaade63481392676c6bbb53a0d6, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731208605815 2024-11-10T03:16:50,519 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting c29ea343310b4db3801e38b22c82f03c, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1731208606477 2024-11-10T03:16:50,519 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting cad986c7afc645a5b578e01c9bf4ab7b, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731208606854 2024-11-10T03:16:50,519 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d5e517c23014f818f98108d8254a81d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1731208606477 2024-11-10T03:16:50,520 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 31fb514dd41a43c58ce6eec10b0d08d0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731208606854 2024-11-10T03:16:50,520 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8045e9cb51c64fe9bc686c5f7bca09a8, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731208608029 2024-11-10T03:16:50,521 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 0aa5edd7d5024eb9866743af026efd58, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731208608031 2024-11-10T03:16:50,539 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f294ea12149d12df042ace2dfb55c011#B#compaction#135 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:16:50,540 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/B/14bbcbf2779b4e48b38ac43f5e02e428 is 50, key is test_row_0/B:col10/1731208608034/Put/seqid=0 2024-11-10T03:16:50,573 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f294ea12149d12df042ace2dfb55c011] 2024-11-10T03:16:50,588 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-10T03:16:50,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:16:50,589 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing f294ea12149d12df042ace2dfb55c011 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-10T03:16:50,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f294ea12149d12df042ace2dfb55c011, store=A 2024-11-10T03:16:50,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:16:50,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f294ea12149d12df042ace2dfb55c011, store=B 2024-11-10T03:16:50,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:16:50,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f294ea12149d12df042ace2dfb55c011, store=C 2024-11-10T03:16:50,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:16:50,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741986_1162 (size=12527) 2024-11-10T03:16:50,609 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111056d7d59643944d89bf2502142eec596f_f294ea12149d12df042ace2dfb55c011 store=[table=TestAcidGuarantees family=A region=f294ea12149d12df042ace2dfb55c011] 2024-11-10T03:16:50,612 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111056d7d59643944d89bf2502142eec596f_f294ea12149d12df042ace2dfb55c011, store=[table=TestAcidGuarantees family=A region=f294ea12149d12df042ace2dfb55c011] 2024-11-10T03:16:50,612 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111056d7d59643944d89bf2502142eec596f_f294ea12149d12df042ace2dfb55c011 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f294ea12149d12df042ace2dfb55c011] 2024-11-10T03:16:50,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-10T03:16:50,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110f0639e8300444c65aea6e1d6adeb5c7e_f294ea12149d12df042ace2dfb55c011 is 50, key is test_row_0/A:col10/1731208609116/Put/seqid=0 2024-11-10T03:16:50,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741987_1163 (size=4469) 2024-11-10T03:16:50,689 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f294ea12149d12df042ace2dfb55c011#A#compaction#136 average throughput is 0.21 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:16:50,689 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/A/c0d3a7083abb4ece9ccd6935a6f0a607 is 175, key is test_row_0/A:col10/1731208608034/Put/seqid=0 2024-11-10T03:16:50,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741988_1164 (size=12304) 2024-11-10T03:16:50,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741989_1165 (size=31481) 2024-11-10T03:16:50,747 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/A/c0d3a7083abb4ece9ccd6935a6f0a607 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/c0d3a7083abb4ece9ccd6935a6f0a607 2024-11-10T03:16:50,769 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f294ea12149d12df042ace2dfb55c011/A of f294ea12149d12df042ace2dfb55c011 into c0d3a7083abb4ece9ccd6935a6f0a607(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:16:50,769 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f294ea12149d12df042ace2dfb55c011: 2024-11-10T03:16:50,769 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., storeName=f294ea12149d12df042ace2dfb55c011/A, priority=12, startTime=1731208610513; duration=0sec 2024-11-10T03:16:50,769 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T03:16:50,769 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f294ea12149d12df042ace2dfb55c011:A 2024-11-10T03:16:50,769 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T03:16:50,777 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48844 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T03:16:50,780 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1541): f294ea12149d12df042ace2dfb55c011/C is initiating minor compaction (all files) 2024-11-10T03:16:50,780 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f294ea12149d12df042ace2dfb55c011/C in TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:16:50,780 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/3aca2c39adad4adea221f8c80be0c51e, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/93e15adcf384445b99e700001c6d0776, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/0deab162ce8d4b3ebbcf348ad6dac0a6, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/5383bb8043d8467fa677ef5c252776c7] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp, totalSize=47.7 K 2024-11-10T03:16:50,781 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3aca2c39adad4adea221f8c80be0c51e, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731208605815 2024-11-10T03:16:50,781 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 93e15adcf384445b99e700001c6d0776, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1731208606477 2024-11-10T03:16:50,783 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0deab162ce8d4b3ebbcf348ad6dac0a6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731208606854 2024-11-10T03:16:50,783 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5383bb8043d8467fa677ef5c252776c7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731208608031 2024-11-10T03:16:50,810 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f294ea12149d12df042ace2dfb55c011#C#compaction#138 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:16:50,811 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/C/d4c320f2ae08487ca6daa1a81f230a02 is 50, key is test_row_0/C:col10/1731208608034/Put/seqid=0 2024-11-10T03:16:50,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741990_1166 (size=12527) 2024-11-10T03:16:50,852 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/C/d4c320f2ae08487ca6daa1a81f230a02 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/d4c320f2ae08487ca6daa1a81f230a02 2024-11-10T03:16:50,864 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f294ea12149d12df042ace2dfb55c011/C of f294ea12149d12df042ace2dfb55c011 into d4c320f2ae08487ca6daa1a81f230a02(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:16:50,864 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f294ea12149d12df042ace2dfb55c011: 2024-11-10T03:16:50,864 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., storeName=f294ea12149d12df042ace2dfb55c011/C, priority=12, startTime=1731208610514; duration=0sec 2024-11-10T03:16:50,864 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:16:50,864 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f294ea12149d12df042ace2dfb55c011:C 2024-11-10T03:16:51,031 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/B/14bbcbf2779b4e48b38ac43f5e02e428 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/14bbcbf2779b4e48b38ac43f5e02e428 2024-11-10T03:16:51,040 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f294ea12149d12df042ace2dfb55c011/B of f294ea12149d12df042ace2dfb55c011 into 14bbcbf2779b4e48b38ac43f5e02e428(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:16:51,040 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f294ea12149d12df042ace2dfb55c011: 2024-11-10T03:16:51,040 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., storeName=f294ea12149d12df042ace2dfb55c011/B, priority=12, startTime=1731208610514; duration=0sec 2024-11-10T03:16:51,041 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:16:51,041 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f294ea12149d12df042ace2dfb55c011:B 2024-11-10T03:16:51,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:51,107 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110f0639e8300444c65aea6e1d6adeb5c7e_f294ea12149d12df042ace2dfb55c011 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110f0639e8300444c65aea6e1d6adeb5c7e_f294ea12149d12df042ace2dfb55c011 2024-11-10T03:16:51,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/A/1524fa518d1247a6b238cce56699e62e, store: [table=TestAcidGuarantees family=A region=f294ea12149d12df042ace2dfb55c011] 2024-11-10T03:16:51,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/A/1524fa518d1247a6b238cce56699e62e is 175, key is test_row_0/A:col10/1731208609116/Put/seqid=0 2024-11-10T03:16:51,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741991_1167 (size=31105) 2024-11-10T03:16:51,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on f294ea12149d12df042ace2dfb55c011 2024-11-10T03:16:51,318 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. as already flushing 2024-11-10T03:16:51,346 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:51,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37914 deadline: 1731208671345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:51,347 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:51,347 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:51,347 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:51,353 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:51,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37948 deadline: 1731208671349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:51,355 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:51,355 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:51,355 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:51,457 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:51,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37914 deadline: 1731208671457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:51,458 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:51,458 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:51,458 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:51,463 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:51,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37948 deadline: 1731208671463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:51,464 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:51,464 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:51,464 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:51,544 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=201, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/A/1524fa518d1247a6b238cce56699e62e 2024-11-10T03:16:51,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/B/96a1b69b41b346d3a51de986f587f83a is 50, key is test_row_0/B:col10/1731208609116/Put/seqid=0 2024-11-10T03:16:51,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741992_1168 (size=12151) 2024-11-10T03:16:51,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-10T03:16:51,664 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:51,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37914 deadline: 1731208671663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:51,665 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:51,665 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:51,665 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:51,677 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:51,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37948 deadline: 1731208671673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:51,678 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:51,678 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:51,678 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:51,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:51,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37914 deadline: 1731208671973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:51,974 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:51,975 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:51,975 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:51,984 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:51,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37948 deadline: 1731208671983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:51,984 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:51,984 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:51,984 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:51,993 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/B/96a1b69b41b346d3a51de986f587f83a 2024-11-10T03:16:52,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/C/67bb0579a8984d0b9b06a2132a9cba1b is 50, key is test_row_0/C:col10/1731208609116/Put/seqid=0 2024-11-10T03:16:52,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741993_1169 (size=12151) 2024-11-10T03:16:52,010 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/C/67bb0579a8984d0b9b06a2132a9cba1b 2024-11-10T03:16:52,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/A/1524fa518d1247a6b238cce56699e62e as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/1524fa518d1247a6b238cce56699e62e 2024-11-10T03:16:52,024 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/1524fa518d1247a6b238cce56699e62e, entries=150, sequenceid=201, filesize=30.4 K 2024-11-10T03:16:52,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/B/96a1b69b41b346d3a51de986f587f83a as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/96a1b69b41b346d3a51de986f587f83a 2024-11-10T03:16:52,032 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/96a1b69b41b346d3a51de986f587f83a, entries=150, sequenceid=201, filesize=11.9 K 2024-11-10T03:16:52,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/C/67bb0579a8984d0b9b06a2132a9cba1b as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/67bb0579a8984d0b9b06a2132a9cba1b 2024-11-10T03:16:52,043 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/67bb0579a8984d0b9b06a2132a9cba1b, entries=150, sequenceid=201, filesize=11.9 K 2024-11-10T03:16:52,044 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for f294ea12149d12df042ace2dfb55c011 in 1456ms, sequenceid=201, compaction requested=false 2024-11-10T03:16:52,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for f294ea12149d12df042ace2dfb55c011: 2024-11-10T03:16:52,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:16:52,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-10T03:16:52,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-10T03:16:52,049 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-10T03:16:52,049 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5650 sec 2024-11-10T03:16:52,051 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 2.5760 sec 2024-11-10T03:16:52,221 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f294ea12149d12df042ace2dfb55c011 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-10T03:16:52,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f294ea12149d12df042ace2dfb55c011, store=A 2024-11-10T03:16:52,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:16:52,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f294ea12149d12df042ace2dfb55c011, store=B 2024-11-10T03:16:52,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:16:52,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f294ea12149d12df042ace2dfb55c011, store=C 2024-11-10T03:16:52,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:16:52,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on f294ea12149d12df042ace2dfb55c011 2024-11-10T03:16:52,262 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111020b65f3c93844b0586468f9227a93f79_f294ea12149d12df042ace2dfb55c011 is 50, key is test_row_0/A:col10/1731208611330/Put/seqid=0 2024-11-10T03:16:52,284 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:52,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37968 deadline: 1731208672275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:52,289 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:52,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37896 deadline: 1731208672285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:52,290 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:52,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37958 deadline: 1731208672285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:52,290 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:52,290 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:52,290 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:52,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:52,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:52,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:52,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:52,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:52,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:52,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741994_1170 (size=12304) 2024-11-10T03:16:52,298 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:52,307 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111020b65f3c93844b0586468f9227a93f79_f294ea12149d12df042ace2dfb55c011 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111020b65f3c93844b0586468f9227a93f79_f294ea12149d12df042ace2dfb55c011 2024-11-10T03:16:52,308 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/A/cf8421c4775b4ce89b65c91472a8f189, store: [table=TestAcidGuarantees family=A region=f294ea12149d12df042ace2dfb55c011] 2024-11-10T03:16:52,309 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/A/cf8421c4775b4ce89b65c91472a8f189 is 175, key is test_row_0/A:col10/1731208611330/Put/seqid=0 2024-11-10T03:16:52,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:52,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37896 deadline: 1731208672393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:52,395 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:52,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:52,395 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:52,395 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:52,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37958 deadline: 1731208672393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:52,395 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:52,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:52,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:52,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741995_1171 (size=31105) 2024-11-10T03:16:52,397 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=218, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/A/cf8421c4775b4ce89b65c91472a8f189 2024-11-10T03:16:52,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:52,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37968 deadline: 1731208672393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:52,399 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:52,399 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:52,399 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:52,430 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/B/60e612dae1374441a48c7efb3ad6103b is 50, key is test_row_0/B:col10/1731208611330/Put/seqid=0 2024-11-10T03:16:52,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741996_1172 (size=12151) 2024-11-10T03:16:52,444 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/B/60e612dae1374441a48c7efb3ad6103b 2024-11-10T03:16:52,455 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/C/17a01aa79d214dac9505e287aabec4a0 is 50, key is test_row_0/C:col10/1731208611330/Put/seqid=0 2024-11-10T03:16:52,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741997_1173 (size=12151) 2024-11-10T03:16:52,490 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:52,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37914 deadline: 1731208672484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:52,491 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:52,491 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:52,491 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:52,491 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/C/17a01aa79d214dac9505e287aabec4a0 2024-11-10T03:16:52,500 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:52,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37948 deadline: 1731208672494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:52,501 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:52,501 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:52,501 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:52,511 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/A/cf8421c4775b4ce89b65c91472a8f189 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/cf8421c4775b4ce89b65c91472a8f189 2024-11-10T03:16:52,520 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/cf8421c4775b4ce89b65c91472a8f189, entries=150, sequenceid=218, filesize=30.4 K 2024-11-10T03:16:52,522 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/B/60e612dae1374441a48c7efb3ad6103b as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/60e612dae1374441a48c7efb3ad6103b 2024-11-10T03:16:52,534 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/60e612dae1374441a48c7efb3ad6103b, entries=150, sequenceid=218, filesize=11.9 K 2024-11-10T03:16:52,544 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/C/17a01aa79d214dac9505e287aabec4a0 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/17a01aa79d214dac9505e287aabec4a0 2024-11-10T03:16:52,551 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/17a01aa79d214dac9505e287aabec4a0, entries=150, sequenceid=218, filesize=11.9 K 2024-11-10T03:16:52,552 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for f294ea12149d12df042ace2dfb55c011 in 331ms, sequenceid=218, compaction requested=true 2024-11-10T03:16:52,552 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f294ea12149d12df042ace2dfb55c011: 2024-11-10T03:16:52,552 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T03:16:52,552 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f294ea12149d12df042ace2dfb55c011:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T03:16:52,553 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:16:52,553 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f294ea12149d12df042ace2dfb55c011:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T03:16:52,553 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:16:52,553 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f294ea12149d12df042ace2dfb55c011:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T03:16:52,553 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T03:16:52,553 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T03:16:52,553 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93691 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T03:16:52,553 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T03:16:52,553 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1541): f294ea12149d12df042ace2dfb55c011/A is initiating minor compaction (all files) 2024-11-10T03:16:52,553 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1541): f294ea12149d12df042ace2dfb55c011/B is initiating minor compaction (all files) 2024-11-10T03:16:52,554 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f294ea12149d12df042ace2dfb55c011/A in TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:16:52,554 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f294ea12149d12df042ace2dfb55c011/B in TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:16:52,554 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/14bbcbf2779b4e48b38ac43f5e02e428, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/96a1b69b41b346d3a51de986f587f83a, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/60e612dae1374441a48c7efb3ad6103b] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp, totalSize=36.0 K 2024-11-10T03:16:52,554 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/c0d3a7083abb4ece9ccd6935a6f0a607, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/1524fa518d1247a6b238cce56699e62e, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/cf8421c4775b4ce89b65c91472a8f189] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp, totalSize=91.5 K 2024-11-10T03:16:52,554 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:16:52,554 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. files: [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/c0d3a7083abb4ece9ccd6935a6f0a607, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/1524fa518d1247a6b238cce56699e62e, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/cf8421c4775b4ce89b65c91472a8f189] 2024-11-10T03:16:52,554 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 14bbcbf2779b4e48b38ac43f5e02e428, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731208608031 2024-11-10T03:16:52,554 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting c0d3a7083abb4ece9ccd6935a6f0a607, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731208608031 2024-11-10T03:16:52,554 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 96a1b69b41b346d3a51de986f587f83a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1731208609116 2024-11-10T03:16:52,554 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1524fa518d1247a6b238cce56699e62e, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1731208609116 2024-11-10T03:16:52,555 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 60e612dae1374441a48c7efb3ad6103b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731208611330 2024-11-10T03:16:52,555 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf8421c4775b4ce89b65c91472a8f189, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731208611330 2024-11-10T03:16:52,566 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f294ea12149d12df042ace2dfb55c011#B#compaction#144 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:16:52,567 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/B/d21fc88b51cf42c3802cf48b86fa9a78 is 50, key is test_row_0/B:col10/1731208611330/Put/seqid=0 2024-11-10T03:16:52,573 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f294ea12149d12df042ace2dfb55c011] 2024-11-10T03:16:52,575 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111023d3589247d94f4abe11a889de06231c_f294ea12149d12df042ace2dfb55c011 store=[table=TestAcidGuarantees family=A region=f294ea12149d12df042ace2dfb55c011] 2024-11-10T03:16:52,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741998_1174 (size=12629) 2024-11-10T03:16:52,578 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111023d3589247d94f4abe11a889de06231c_f294ea12149d12df042ace2dfb55c011, store=[table=TestAcidGuarantees family=A region=f294ea12149d12df042ace2dfb55c011] 2024-11-10T03:16:52,578 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111023d3589247d94f4abe11a889de06231c_f294ea12149d12df042ace2dfb55c011 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f294ea12149d12df042ace2dfb55c011] 2024-11-10T03:16:52,584 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/B/d21fc88b51cf42c3802cf48b86fa9a78 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/d21fc88b51cf42c3802cf48b86fa9a78 2024-11-10T03:16:52,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741999_1175 (size=4469) 2024-11-10T03:16:52,590 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f294ea12149d12df042ace2dfb55c011/B of f294ea12149d12df042ace2dfb55c011 into d21fc88b51cf42c3802cf48b86fa9a78(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:16:52,590 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f294ea12149d12df042ace2dfb55c011: 2024-11-10T03:16:52,590 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., storeName=f294ea12149d12df042ace2dfb55c011/B, priority=13, startTime=1731208612553; duration=0sec 2024-11-10T03:16:52,590 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T03:16:52,590 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f294ea12149d12df042ace2dfb55c011:B 2024-11-10T03:16:52,590 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T03:16:52,591 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T03:16:52,592 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1541): f294ea12149d12df042ace2dfb55c011/C is initiating minor compaction (all files) 2024-11-10T03:16:52,592 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f294ea12149d12df042ace2dfb55c011/C in TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:16:52,592 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/d4c320f2ae08487ca6daa1a81f230a02, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/67bb0579a8984d0b9b06a2132a9cba1b, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/17a01aa79d214dac9505e287aabec4a0] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp, totalSize=36.0 K 2024-11-10T03:16:52,592 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting d4c320f2ae08487ca6daa1a81f230a02, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731208608031 2024-11-10T03:16:52,592 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 67bb0579a8984d0b9b06a2132a9cba1b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1731208609116 2024-11-10T03:16:52,593 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 17a01aa79d214dac9505e287aabec4a0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731208611330 2024-11-10T03:16:52,602 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f294ea12149d12df042ace2dfb55c011#C#compaction#146 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:16:52,602 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/C/e2e01ba5043445babdf65324e15dd6e8 is 50, key is test_row_0/C:col10/1731208611330/Put/seqid=0 2024-11-10T03:16:52,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on f294ea12149d12df042ace2dfb55c011 2024-11-10T03:16:52,606 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f294ea12149d12df042ace2dfb55c011 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-10T03:16:52,607 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f294ea12149d12df042ace2dfb55c011, store=A 2024-11-10T03:16:52,607 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:16:52,607 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f294ea12149d12df042ace2dfb55c011, store=B 2024-11-10T03:16:52,607 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:16:52,607 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f294ea12149d12df042ace2dfb55c011, store=C 2024-11-10T03:16:52,607 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:16:52,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742000_1176 (size=12629) 2024-11-10T03:16:52,617 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110605ab5b011074c139c759ec5db7d5fc0_f294ea12149d12df042ace2dfb55c011 is 50, key is test_row_0/A:col10/1731208612280/Put/seqid=0 2024-11-10T03:16:52,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742001_1177 (size=14794) 2024-11-10T03:16:52,633 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:52,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37968 deadline: 1731208672628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:52,636 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:52,636 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:52,636 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:52,638 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:52,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37958 deadline: 1731208672633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:52,638 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:52,639 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:52,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37896 deadline: 1731208672634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:52,639 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:52,639 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:52,639 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:52,639 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:52,639 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:52,745 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:52,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37968 deadline: 1731208672743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:52,745 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:52,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37958 deadline: 1731208672743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:52,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:52,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:52,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:52,746 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:52,746 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:52,746 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:52,746 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:52,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37896 deadline: 1731208672743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:52,746 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:52,746 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:52,746 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:52,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:52,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37968 deadline: 1731208672953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:52,957 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:52,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37958 deadline: 1731208672953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:52,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:52,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37896 deadline: 1731208672953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:52,958 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:52,958 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:52,958 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:52,958 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:52,958 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:52,958 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:52,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:52,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:52,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:52,994 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f294ea12149d12df042ace2dfb55c011#A#compaction#145 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:16:52,995 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/A/c2eda5347b6f417baa61a5abd3987da7 is 175, key is test_row_0/A:col10/1731208611330/Put/seqid=0 2024-11-10T03:16:53,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742002_1178 (size=31583) 2024-11-10T03:16:53,013 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/C/e2e01ba5043445babdf65324e15dd6e8 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/e2e01ba5043445babdf65324e15dd6e8 2024-11-10T03:16:53,021 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f294ea12149d12df042ace2dfb55c011/C of f294ea12149d12df042ace2dfb55c011 into e2e01ba5043445babdf65324e15dd6e8(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:16:53,021 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f294ea12149d12df042ace2dfb55c011: 2024-11-10T03:16:53,021 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., storeName=f294ea12149d12df042ace2dfb55c011/C, priority=13, startTime=1731208612553; duration=0sec 2024-11-10T03:16:53,021 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:16:53,021 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f294ea12149d12df042ace2dfb55c011:C 2024-11-10T03:16:53,024 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:53,032 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110605ab5b011074c139c759ec5db7d5fc0_f294ea12149d12df042ace2dfb55c011 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110605ab5b011074c139c759ec5db7d5fc0_f294ea12149d12df042ace2dfb55c011 2024-11-10T03:16:53,036 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/A/7e5c546522dc4a8890790c771a6e103e, store: [table=TestAcidGuarantees family=A region=f294ea12149d12df042ace2dfb55c011] 2024-11-10T03:16:53,037 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/A/7e5c546522dc4a8890790c771a6e103e is 175, key is test_row_0/A:col10/1731208612280/Put/seqid=0 2024-11-10T03:16:53,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742003_1179 (size=39749) 2024-11-10T03:16:53,057 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=239, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/A/7e5c546522dc4a8890790c771a6e103e 2024-11-10T03:16:53,068 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/B/bb654c7282eb4cd7bdc37bf3c831d7fc is 50, key is test_row_0/B:col10/1731208612280/Put/seqid=0 2024-11-10T03:16:53,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742004_1180 (size=12151) 2024-11-10T03:16:53,264 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:53,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37958 deadline: 1731208673263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:53,264 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:53,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:53,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:53,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37968 deadline: 1731208673263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:53,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:53,264 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:53,264 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:53,265 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:53,265 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:53,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37896 deadline: 1731208673263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:53,265 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:53,265 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:53,265 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:53,418 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/A/c2eda5347b6f417baa61a5abd3987da7 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/c2eda5347b6f417baa61a5abd3987da7 2024-11-10T03:16:53,424 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f294ea12149d12df042ace2dfb55c011/A of f294ea12149d12df042ace2dfb55c011 into c2eda5347b6f417baa61a5abd3987da7(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:16:53,424 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f294ea12149d12df042ace2dfb55c011: 2024-11-10T03:16:53,424 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., storeName=f294ea12149d12df042ace2dfb55c011/A, priority=13, startTime=1731208612552; duration=0sec 2024-11-10T03:16:53,424 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:16:53,424 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f294ea12149d12df042ace2dfb55c011:A 2024-11-10T03:16:53,506 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:53,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37914 deadline: 1731208673503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:53,506 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 394d2222f724,38139,1731208572054 for 'test_row_0' in f294ea12149d12df042ace2dfb55c011 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T03:16:53,507 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:53,507 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:53,507 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:53,514 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/B/bb654c7282eb4cd7bdc37bf3c831d7fc 2024-11-10T03:16:53,515 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:53,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37948 deadline: 1731208673513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:53,516 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 394d2222f724,38139,1731208572054 for 'test_row_1' in f294ea12149d12df042ace2dfb55c011 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T03:16:53,516 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:53,516 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:53,516 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:53,522 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/C/128133b7282b44ff9ce929eb8769d7f0 is 50, key is test_row_0/C:col10/1731208612280/Put/seqid=0 2024-11-10T03:16:53,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742005_1181 (size=12151) 2024-11-10T03:16:53,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-10T03:16:53,654 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T03:16:53,654 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T03:16:53,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-10T03:16:53,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-10T03:16:53,658 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T03:16:53,658 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T03:16:53,659 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T03:16:53,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-10T03:16:53,773 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:53,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37968 deadline: 1731208673773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:53,774 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:53,774 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:53,774 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:53,775 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:53,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37958 deadline: 1731208673775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:53,776 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:53,776 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:53,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37896 deadline: 1731208673775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:53,776 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:53,776 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:53,776 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:53,776 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:53,776 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:53,811 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-10T03:16:53,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:16:53,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. as already flushing 2024-11-10T03:16:53,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:16:53,812 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:53,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:53,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:53,929 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/C/128133b7282b44ff9ce929eb8769d7f0 2024-11-10T03:16:53,934 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/A/7e5c546522dc4a8890790c771a6e103e as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/7e5c546522dc4a8890790c771a6e103e 2024-11-10T03:16:53,938 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/7e5c546522dc4a8890790c771a6e103e, entries=200, sequenceid=239, filesize=38.8 K 2024-11-10T03:16:53,939 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/B/bb654c7282eb4cd7bdc37bf3c831d7fc as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/bb654c7282eb4cd7bdc37bf3c831d7fc 2024-11-10T03:16:53,944 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/bb654c7282eb4cd7bdc37bf3c831d7fc, entries=150, sequenceid=239, filesize=11.9 K 2024-11-10T03:16:53,945 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/C/128133b7282b44ff9ce929eb8769d7f0 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/128133b7282b44ff9ce929eb8769d7f0 2024-11-10T03:16:53,949 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/128133b7282b44ff9ce929eb8769d7f0, entries=150, sequenceid=239, filesize=11.9 K 2024-11-10T03:16:53,950 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for f294ea12149d12df042ace2dfb55c011 in 1344ms, sequenceid=239, compaction requested=false 2024-11-10T03:16:53,950 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f294ea12149d12df042ace2dfb55c011: 2024-11-10T03:16:53,965 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-10T03:16:53,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:16:53,965 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing f294ea12149d12df042ace2dfb55c011 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-10T03:16:53,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f294ea12149d12df042ace2dfb55c011, store=A 2024-11-10T03:16:53,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:16:53,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f294ea12149d12df042ace2dfb55c011, store=B 2024-11-10T03:16:53,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:16:53,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f294ea12149d12df042ace2dfb55c011, store=C 2024-11-10T03:16:53,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:16:53,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111034f61ad1ea274020afcb6eb84d2f9028_f294ea12149d12df042ace2dfb55c011 is 50, key is test_row_0/A:col10/1731208612631/Put/seqid=0 2024-11-10T03:16:53,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-10T03:16:53,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742006_1182 (size=12304) 2024-11-10T03:16:53,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:53,987 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111034f61ad1ea274020afcb6eb84d2f9028_f294ea12149d12df042ace2dfb55c011 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111034f61ad1ea274020afcb6eb84d2f9028_f294ea12149d12df042ace2dfb55c011 2024-11-10T03:16:53,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/A/e38a362af6ae4832804fde301289c786, store: [table=TestAcidGuarantees family=A region=f294ea12149d12df042ace2dfb55c011] 2024-11-10T03:16:53,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/A/e38a362af6ae4832804fde301289c786 is 175, key is test_row_0/A:col10/1731208612631/Put/seqid=0 2024-11-10T03:16:54,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742007_1183 (size=31105) 2024-11-10T03:16:54,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-10T03:16:54,426 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=254, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/A/e38a362af6ae4832804fde301289c786 2024-11-10T03:16:54,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/B/a3db52a6578b41f7886373720b94b32b is 50, key is test_row_0/B:col10/1731208612631/Put/seqid=0 2024-11-10T03:16:54,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742008_1184 (size=12151) 2024-11-10T03:16:54,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on f294ea12149d12df042ace2dfb55c011 2024-11-10T03:16:54,785 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. as already flushing 2024-11-10T03:16:54,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-10T03:16:54,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:54,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37896 deadline: 1731208674816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:54,823 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:54,823 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:54,823 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:54,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:54,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37958 deadline: 1731208674821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:54,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:54,828 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:54,828 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:54,828 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:54,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37968 deadline: 1731208674824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:54,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:54,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:54,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:54,838 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/B/a3db52a6578b41f7886373720b94b32b 2024-11-10T03:16:54,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/C/8ee9eaf43087453c89d2dec15bc50129 is 50, key is test_row_0/C:col10/1731208612631/Put/seqid=0 2024-11-10T03:16:54,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742009_1185 (size=12151) 2024-11-10T03:16:54,900 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/C/8ee9eaf43087453c89d2dec15bc50129 2024-11-10T03:16:54,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/A/e38a362af6ae4832804fde301289c786 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/e38a362af6ae4832804fde301289c786 2024-11-10T03:16:54,911 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/e38a362af6ae4832804fde301289c786, entries=150, sequenceid=254, filesize=30.4 K 2024-11-10T03:16:54,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/B/a3db52a6578b41f7886373720b94b32b as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/a3db52a6578b41f7886373720b94b32b 2024-11-10T03:16:54,919 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/a3db52a6578b41f7886373720b94b32b, entries=150, sequenceid=254, filesize=11.9 K 2024-11-10T03:16:54,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/C/8ee9eaf43087453c89d2dec15bc50129 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/8ee9eaf43087453c89d2dec15bc50129 2024-11-10T03:16:54,925 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/8ee9eaf43087453c89d2dec15bc50129, entries=150, sequenceid=254, filesize=11.9 K 2024-11-10T03:16:54,926 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for f294ea12149d12df042ace2dfb55c011 in 961ms, sequenceid=254, compaction requested=true 2024-11-10T03:16:54,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for f294ea12149d12df042ace2dfb55c011: 2024-11-10T03:16:54,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:16:54,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-10T03:16:54,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-10T03:16:54,931 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-10T03:16:54,931 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2700 sec 2024-11-10T03:16:54,932 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 1.2770 sec 2024-11-10T03:16:54,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on f294ea12149d12df042ace2dfb55c011 2024-11-10T03:16:54,937 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f294ea12149d12df042ace2dfb55c011 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-10T03:16:54,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f294ea12149d12df042ace2dfb55c011, store=A 2024-11-10T03:16:54,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:16:54,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f294ea12149d12df042ace2dfb55c011, store=B 2024-11-10T03:16:54,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:16:54,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f294ea12149d12df042ace2dfb55c011, store=C 2024-11-10T03:16:54,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:16:54,960 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110dc3c9ebae322464189b05560544cf756_f294ea12149d12df042ace2dfb55c011 is 50, key is test_row_0/A:col10/1731208614806/Put/seqid=0 2024-11-10T03:16:54,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:54,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37896 deadline: 1731208674957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:54,965 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:54,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37968 deadline: 1731208674959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:54,966 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:54,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37958 deadline: 1731208674966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:54,966 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:54,966 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:54,966 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:54,966 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:54,966 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:54,966 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:54,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:54,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:54,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:54,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742010_1186 (size=14994) 2024-11-10T03:16:55,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:55,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37968 deadline: 1731208675073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:55,074 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:55,074 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:55,074 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:55,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37896 deadline: 1731208675073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:55,074 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:55,074 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:55,074 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:55,074 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:55,074 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:55,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37958 deadline: 1731208675073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:55,075 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:55,075 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:55,075 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:55,286 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:55,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37968 deadline: 1731208675282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:55,287 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:55,287 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:55,287 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:55,287 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:55,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37896 deadline: 1731208675282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:55,287 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:55,287 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:55,287 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:55,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37958 deadline: 1731208675282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:55,287 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:55,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:55,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:55,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:55,374 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:55,379 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110dc3c9ebae322464189b05560544cf756_f294ea12149d12df042ace2dfb55c011 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110dc3c9ebae322464189b05560544cf756_f294ea12149d12df042ace2dfb55c011 2024-11-10T03:16:55,380 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/A/d7a444cd57574b61b9d1c72a79eb207d, store: [table=TestAcidGuarantees family=A region=f294ea12149d12df042ace2dfb55c011] 2024-11-10T03:16:55,381 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/A/d7a444cd57574b61b9d1c72a79eb207d is 175, key is test_row_0/A:col10/1731208614806/Put/seqid=0 2024-11-10T03:16:55,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742011_1187 (size=39949) 2024-11-10T03:16:55,399 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=274, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/A/d7a444cd57574b61b9d1c72a79eb207d 2024-11-10T03:16:55,406 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/B/5cd6128793ef415a944938f8659be45f is 50, key is test_row_0/B:col10/1731208614806/Put/seqid=0 2024-11-10T03:16:55,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742012_1188 (size=12301) 2024-11-10T03:16:55,523 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:55,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37914 deadline: 1731208675522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:55,523 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 394d2222f724,38139,1731208572054 for 'test_row_0' in f294ea12149d12df042ace2dfb55c011 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T03:16:55,524 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:55,524 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:55,524 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:55,543 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:55,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37948 deadline: 1731208675543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:55,544 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 394d2222f724,38139,1731208572054 for 'test_row_1' in f294ea12149d12df042ace2dfb55c011 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T03:16:55,544 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:55,544 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:55,544 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:55,597 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:55,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37968 deadline: 1731208675593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:55,597 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:55,598 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:55,597 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:55,598 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:55,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37896 deadline: 1731208675593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:55,598 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:55,598 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:55,598 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:55,599 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:55,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37958 deadline: 1731208675594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:55,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:55,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:55,603 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:55,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-10T03:16:55,803 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T03:16:55,804 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T03:16:55,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-10T03:16:55,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-10T03:16:55,807 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T03:16:55,807 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T03:16:55,808 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T03:16:55,811 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/B/5cd6128793ef415a944938f8659be45f 2024-11-10T03:16:55,825 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/C/137e6978efea4f138aeac3c3d511f789 is 50, key is test_row_0/C:col10/1731208614806/Put/seqid=0 2024-11-10T03:16:55,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742013_1189 (size=12301) 2024-11-10T03:16:55,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-10T03:16:55,961 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-10T03:16:55,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:16:55,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. as already flushing 2024-11-10T03:16:55,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:16:55,962 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:55,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:55,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:56,114 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-10T03:16:56,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:16:56,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. as already flushing 2024-11-10T03:16:56,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:16:56,115 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:56,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:56,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:56,117 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:56,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37968 deadline: 1731208676113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:56,118 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:56,118 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:56,118 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:56,118 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:56,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37896 deadline: 1731208676113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:56,118 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:56,118 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:56,118 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:56,118 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:56,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37958 deadline: 1731208676113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:56,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:56,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:56,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:56,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-10T03:16:56,233 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/C/137e6978efea4f138aeac3c3d511f789 2024-11-10T03:16:56,240 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/A/d7a444cd57574b61b9d1c72a79eb207d as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/d7a444cd57574b61b9d1c72a79eb207d 2024-11-10T03:16:56,246 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/d7a444cd57574b61b9d1c72a79eb207d, entries=200, sequenceid=274, filesize=39.0 K 2024-11-10T03:16:56,248 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/B/5cd6128793ef415a944938f8659be45f as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/5cd6128793ef415a944938f8659be45f 2024-11-10T03:16:56,256 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/5cd6128793ef415a944938f8659be45f, entries=150, sequenceid=274, filesize=12.0 K 2024-11-10T03:16:56,258 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/C/137e6978efea4f138aeac3c3d511f789 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/137e6978efea4f138aeac3c3d511f789 2024-11-10T03:16:56,264 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/137e6978efea4f138aeac3c3d511f789, entries=150, sequenceid=274, filesize=12.0 K 2024-11-10T03:16:56,265 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for f294ea12149d12df042ace2dfb55c011 in 1327ms, sequenceid=274, compaction requested=true 2024-11-10T03:16:56,265 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f294ea12149d12df042ace2dfb55c011: 2024-11-10T03:16:56,265 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f294ea12149d12df042ace2dfb55c011:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T03:16:56,265 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T03:16:56,265 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:16:56,265 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f294ea12149d12df042ace2dfb55c011:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T03:16:56,265 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:16:56,265 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f294ea12149d12df042ace2dfb55c011:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T03:16:56,265 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T03:16:56,265 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T03:16:56,267 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-10T03:16:56,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:16:56,268 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing f294ea12149d12df042ace2dfb55c011 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-10T03:16:56,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f294ea12149d12df042ace2dfb55c011, store=A 2024-11-10T03:16:56,268 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142386 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T03:16:56,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:16:56,268 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49232 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T03:16:56,268 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1541): f294ea12149d12df042ace2dfb55c011/A is initiating minor compaction (all files) 2024-11-10T03:16:56,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f294ea12149d12df042ace2dfb55c011, store=B 2024-11-10T03:16:56,268 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1541): f294ea12149d12df042ace2dfb55c011/B is initiating minor compaction (all files) 2024-11-10T03:16:56,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:16:56,268 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f294ea12149d12df042ace2dfb55c011/A in TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:16:56,268 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f294ea12149d12df042ace2dfb55c011/B in TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:16:56,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f294ea12149d12df042ace2dfb55c011, store=C 2024-11-10T03:16:56,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:16:56,268 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/d21fc88b51cf42c3802cf48b86fa9a78, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/bb654c7282eb4cd7bdc37bf3c831d7fc, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/a3db52a6578b41f7886373720b94b32b, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/5cd6128793ef415a944938f8659be45f] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp, totalSize=48.1 K 2024-11-10T03:16:56,268 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/c2eda5347b6f417baa61a5abd3987da7, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/7e5c546522dc4a8890790c771a6e103e, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/e38a362af6ae4832804fde301289c786, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/d7a444cd57574b61b9d1c72a79eb207d] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp, totalSize=139.0 K 2024-11-10T03:16:56,268 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:16:56,268 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. files: [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/c2eda5347b6f417baa61a5abd3987da7, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/7e5c546522dc4a8890790c771a6e103e, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/e38a362af6ae4832804fde301289c786, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/d7a444cd57574b61b9d1c72a79eb207d] 2024-11-10T03:16:56,269 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting c2eda5347b6f417baa61a5abd3987da7, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731208611330 2024-11-10T03:16:56,269 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting d21fc88b51cf42c3802cf48b86fa9a78, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731208611330 2024-11-10T03:16:56,269 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting bb654c7282eb4cd7bdc37bf3c831d7fc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731208612280 2024-11-10T03:16:56,269 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e5c546522dc4a8890790c771a6e103e, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731208612273 2024-11-10T03:16:56,270 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting a3db52a6578b41f7886373720b94b32b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731208612614 2024-11-10T03:16:56,270 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting e38a362af6ae4832804fde301289c786, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731208612614 2024-11-10T03:16:56,270 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 5cd6128793ef415a944938f8659be45f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731208614806 2024-11-10T03:16:56,272 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting d7a444cd57574b61b9d1c72a79eb207d, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731208614806 2024-11-10T03:16:56,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411106d99db466f3d47489c0c1c4a43f2405e_f294ea12149d12df042ace2dfb55c011 is 50, key is test_row_0/A:col10/1731208614964/Put/seqid=0 2024-11-10T03:16:56,316 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f294ea12149d12df042ace2dfb55c011#B#compaction#157 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:16:56,317 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/B/d2e10726678f411d91cfd9d550a14209 is 50, key is test_row_0/B:col10/1731208614806/Put/seqid=0 2024-11-10T03:16:56,325 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f294ea12149d12df042ace2dfb55c011] 2024-11-10T03:16:56,329 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241110c9579d95aa1444269765eb7dc2a1762d_f294ea12149d12df042ace2dfb55c011 store=[table=TestAcidGuarantees family=A region=f294ea12149d12df042ace2dfb55c011] 2024-11-10T03:16:56,332 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241110c9579d95aa1444269765eb7dc2a1762d_f294ea12149d12df042ace2dfb55c011, store=[table=TestAcidGuarantees family=A region=f294ea12149d12df042ace2dfb55c011] 2024-11-10T03:16:56,332 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110c9579d95aa1444269765eb7dc2a1762d_f294ea12149d12df042ace2dfb55c011 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f294ea12149d12df042ace2dfb55c011] 2024-11-10T03:16:56,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742015_1191 (size=12915) 2024-11-10T03:16:56,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742014_1190 (size=12454) 2024-11-10T03:16:56,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742016_1192 (size=4469) 2024-11-10T03:16:56,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-10T03:16:56,658 INFO [master/394d2222f724:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-10T03:16:56,658 INFO [master/394d2222f724:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-10T03:16:56,763 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/B/d2e10726678f411d91cfd9d550a14209 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/d2e10726678f411d91cfd9d550a14209 2024-11-10T03:16:56,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,769 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411106d99db466f3d47489c0c1c4a43f2405e_f294ea12149d12df042ace2dfb55c011 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411106d99db466f3d47489c0c1c4a43f2405e_f294ea12149d12df042ace2dfb55c011 2024-11-10T03:16:56,770 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f294ea12149d12df042ace2dfb55c011/B of f294ea12149d12df042ace2dfb55c011 into d2e10726678f411d91cfd9d550a14209(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:16:56,770 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f294ea12149d12df042ace2dfb55c011: 2024-11-10T03:16:56,770 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., storeName=f294ea12149d12df042ace2dfb55c011/B, priority=12, startTime=1731208616265; duration=0sec 2024-11-10T03:16:56,770 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T03:16:56,770 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f294ea12149d12df042ace2dfb55c011:B 2024-11-10T03:16:56,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/A/6a9376887ebb42bd82b58762ca9ba345, store: [table=TestAcidGuarantees family=A region=f294ea12149d12df042ace2dfb55c011] 2024-11-10T03:16:56,770 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T03:16:56,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/A/6a9376887ebb42bd82b58762ca9ba345 is 175, key is test_row_0/A:col10/1731208614964/Put/seqid=0 2024-11-10T03:16:56,773 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49232 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T03:16:56,773 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1541): f294ea12149d12df042ace2dfb55c011/C is initiating minor compaction (all files) 2024-11-10T03:16:56,773 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f294ea12149d12df042ace2dfb55c011/C in TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:16:56,773 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/e2e01ba5043445babdf65324e15dd6e8, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/128133b7282b44ff9ce929eb8769d7f0, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/8ee9eaf43087453c89d2dec15bc50129, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/137e6978efea4f138aeac3c3d511f789] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp, totalSize=48.1 K 2024-11-10T03:16:56,774 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting e2e01ba5043445babdf65324e15dd6e8, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731208611330 2024-11-10T03:16:56,775 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f294ea12149d12df042ace2dfb55c011#A#compaction#158 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:16:56,776 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/A/bcc9230d27b14b59b579b86057f85d6f is 175, key is test_row_0/A:col10/1731208614806/Put/seqid=0 2024-11-10T03:16:56,776 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 128133b7282b44ff9ce929eb8769d7f0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731208612280 2024-11-10T03:16:56,780 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ee9eaf43087453c89d2dec15bc50129, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731208612614 2024-11-10T03:16:56,780 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 137e6978efea4f138aeac3c3d511f789, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731208614806 2024-11-10T03:16:56,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742018_1194 (size=31869) 2024-11-10T03:16:56,800 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f294ea12149d12df042ace2dfb55c011#C#compaction#159 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:16:56,801 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/C/8a690cb9d0f84df4ac4d2c37e75893ef is 50, key is test_row_0/C:col10/1731208614806/Put/seqid=0 2024-11-10T03:16:56,826 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/A/bcc9230d27b14b59b579b86057f85d6f as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/bcc9230d27b14b59b579b86057f85d6f 2024-11-10T03:16:56,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742017_1193 (size=31255) 2024-11-10T03:16:56,835 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=286, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/A/6a9376887ebb42bd82b58762ca9ba345 2024-11-10T03:16:56,836 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f294ea12149d12df042ace2dfb55c011/A of f294ea12149d12df042ace2dfb55c011 into bcc9230d27b14b59b579b86057f85d6f(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:16:56,836 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f294ea12149d12df042ace2dfb55c011: 2024-11-10T03:16:56,836 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., storeName=f294ea12149d12df042ace2dfb55c011/A, priority=12, startTime=1731208616265; duration=0sec 2024-11-10T03:16:56,836 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:16:56,836 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f294ea12149d12df042ace2dfb55c011:A 2024-11-10T03:16:56,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742019_1195 (size=12915) 2024-11-10T03:16:56,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/B/12332906f4fa4559ba1d1dcb7d756d47 is 50, key is test_row_0/B:col10/1731208614964/Put/seqid=0 2024-11-10T03:16:56,854 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/C/8a690cb9d0f84df4ac4d2c37e75893ef as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/8a690cb9d0f84df4ac4d2c37e75893ef 2024-11-10T03:16:56,862 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f294ea12149d12df042ace2dfb55c011/C of f294ea12149d12df042ace2dfb55c011 into 8a690cb9d0f84df4ac4d2c37e75893ef(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:16:56,862 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f294ea12149d12df042ace2dfb55c011: 2024-11-10T03:16:56,862 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., storeName=f294ea12149d12df042ace2dfb55c011/C, priority=12, startTime=1731208616265; duration=0sec 2024-11-10T03:16:56,862 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:16:56,862 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f294ea12149d12df042ace2dfb55c011:C 2024-11-10T03:16:56,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742020_1196 (size=12301) 2024-11-10T03:16:56,888 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/B/12332906f4fa4559ba1d1dcb7d756d47 2024-11-10T03:16:56,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/C/5bc9d89143594d0bb42e955ec7d8114c is 50, key is test_row_0/C:col10/1731208614964/Put/seqid=0 2024-11-10T03:16:56,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742021_1197 (size=12301) 2024-11-10T03:16:56,920 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/C/5bc9d89143594d0bb42e955ec7d8114c 2024-11-10T03:16:56,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/A/6a9376887ebb42bd82b58762ca9ba345 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/6a9376887ebb42bd82b58762ca9ba345 2024-11-10T03:16:56,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-10T03:16:56,945 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/6a9376887ebb42bd82b58762ca9ba345, entries=150, sequenceid=286, filesize=30.5 K 2024-11-10T03:16:56,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/B/12332906f4fa4559ba1d1dcb7d756d47 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/12332906f4fa4559ba1d1dcb7d756d47 2024-11-10T03:16:56,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,956 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/12332906f4fa4559ba1d1dcb7d756d47, entries=150, sequenceid=286, filesize=12.0 K 2024-11-10T03:16:56,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/C/5bc9d89143594d0bb42e955ec7d8114c as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/5bc9d89143594d0bb42e955ec7d8114c 2024-11-10T03:16:56,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,963 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/5bc9d89143594d0bb42e955ec7d8114c, entries=150, sequenceid=286, filesize=12.0 K 2024-11-10T03:16:56,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,964 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=0 B/0 for f294ea12149d12df042ace2dfb55c011 in 696ms, sequenceid=286, compaction requested=false 2024-11-10T03:16:56,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for f294ea12149d12df042ace2dfb55c011: 2024-11-10T03:16:56,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:16:56,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-10T03:16:56,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-10T03:16:56,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,968 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-10T03:16:56,968 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1570 sec 2024-11-10T03:16:56,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,973 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 1.1670 sec 2024-11-10T03:16:56,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:56,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,197 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f294ea12149d12df042ace2dfb55c011 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-10T03:16:57,197 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f294ea12149d12df042ace2dfb55c011, store=A 2024-11-10T03:16:57,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,197 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:16:57,197 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f294ea12149d12df042ace2dfb55c011, store=B 2024-11-10T03:16:57,197 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:16:57,197 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f294ea12149d12df042ace2dfb55c011, store=C 2024-11-10T03:16:57,197 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:16:57,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on f294ea12149d12df042ace2dfb55c011 2024-11-10T03:16:57,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,227 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411109f8b2372209b42c683f8216f2c5b121b_f294ea12149d12df042ace2dfb55c011 is 50, key is test_row_0/A:col10/1731208617187/Put/seqid=0 2024-11-10T03:16:57,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742022_1198 (size=22618) 2024-11-10T03:16:57,247 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,252 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411109f8b2372209b42c683f8216f2c5b121b_f294ea12149d12df042ace2dfb55c011 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411109f8b2372209b42c683f8216f2c5b121b_f294ea12149d12df042ace2dfb55c011 2024-11-10T03:16:57,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,253 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/A/0690eb4c09d04350b1ca995efd66b193, store: [table=TestAcidGuarantees family=A region=f294ea12149d12df042ace2dfb55c011] 2024-11-10T03:16:57,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,254 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/A/0690eb4c09d04350b1ca995efd66b193 is 175, key is test_row_0/A:col10/1731208617187/Put/seqid=0 2024-11-10T03:16:57,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742023_1199 (size=66023) 2024-11-10T03:16:57,266 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=299, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/A/0690eb4c09d04350b1ca995efd66b193 2024-11-10T03:16:57,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,281 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/B/728939e45b6447189c598546d97266ba is 50, key is test_row_0/B:col10/1731208617187/Put/seqid=0 2024-11-10T03:16:57,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:57,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742024_1200 (size=22065) 2024-11-10T03:16:57,317 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/B/728939e45b6447189c598546d97266ba 2024-11-10T03:16:57,330 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/C/abd12603d8a944bbac02793dd577f871 is 50, key is test_row_0/C:col10/1731208617187/Put/seqid=0 2024-11-10T03:16:57,333 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731208617290, max timestamp=1731208617329Num uniques 859; 2024-11-10T03:16:57,333 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731208617290, max timestamp=1731208617329Num uniques 859; 2024-11-10T03:16:57,333 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731208617290, max timestamp=1731208617329Num uniques 859; 2024-11-10T03:16:57,333 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T03:16:57,333 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T03:16:57,333 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T03:16:57,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742025_1201 (size=12301) 2024-11-10T03:16:57,380 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:57,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37968 deadline: 1731208677374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:57,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:57,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:57,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:57,387 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:57,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37958 deadline: 1731208677381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:57,387 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:57,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37896 deadline: 1731208677381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:57,388 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:57,388 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:57,388 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:57,388 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:57,388 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:57,388 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:57,487 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:57,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37968 deadline: 1731208677483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:57,487 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:57,487 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:57,487 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:57,494 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:57,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37958 deadline: 1731208677493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:57,494 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:57,494 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:57,494 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:57,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37896 deadline: 1731208677493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:57,494 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:57,495 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:57,495 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:57,495 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:57,717 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:57,717 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:57,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37968 deadline: 1731208677701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:57,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37958 deadline: 1731208677704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:57,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:57,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37896 deadline: 1731208677708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:57,718 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:57,718 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:57,718 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:57,720 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:57,720 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:57,720 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:57,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:57,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:57,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:57,763 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/C/abd12603d8a944bbac02793dd577f871 2024-11-10T03:16:57,776 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/A/0690eb4c09d04350b1ca995efd66b193 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/0690eb4c09d04350b1ca995efd66b193 2024-11-10T03:16:57,792 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/0690eb4c09d04350b1ca995efd66b193, entries=350, sequenceid=299, filesize=64.5 K 2024-11-10T03:16:57,793 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/B/728939e45b6447189c598546d97266ba as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/728939e45b6447189c598546d97266ba 2024-11-10T03:16:57,801 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/728939e45b6447189c598546d97266ba, entries=350, sequenceid=299, filesize=21.5 K 2024-11-10T03:16:57,807 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/C/abd12603d8a944bbac02793dd577f871 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/abd12603d8a944bbac02793dd577f871 2024-11-10T03:16:57,813 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/abd12603d8a944bbac02793dd577f871, entries=150, sequenceid=299, filesize=12.0 K 2024-11-10T03:16:57,816 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for f294ea12149d12df042ace2dfb55c011 in 620ms, sequenceid=299, compaction requested=true 2024-11-10T03:16:57,817 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f294ea12149d12df042ace2dfb55c011: 2024-11-10T03:16:57,817 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T03:16:57,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f294ea12149d12df042ace2dfb55c011:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T03:16:57,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:16:57,817 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T03:16:57,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f294ea12149d12df042ace2dfb55c011:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T03:16:57,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:16:57,818 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f294ea12149d12df042ace2dfb55c011:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T03:16:57,818 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T03:16:57,821 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 47281 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T03:16:57,821 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1541): f294ea12149d12df042ace2dfb55c011/B is initiating minor compaction (all files) 2024-11-10T03:16:57,821 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f294ea12149d12df042ace2dfb55c011/B in TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:16:57,821 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/d2e10726678f411d91cfd9d550a14209, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/12332906f4fa4559ba1d1dcb7d756d47, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/728939e45b6447189c598546d97266ba] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp, totalSize=46.2 K 2024-11-10T03:16:57,821 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 129147 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T03:16:57,821 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1541): f294ea12149d12df042ace2dfb55c011/A is initiating minor compaction (all files) 2024-11-10T03:16:57,821 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f294ea12149d12df042ace2dfb55c011/A in TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:16:57,821 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/bcc9230d27b14b59b579b86057f85d6f, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/6a9376887ebb42bd82b58762ca9ba345, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/0690eb4c09d04350b1ca995efd66b193] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp, totalSize=126.1 K 2024-11-10T03:16:57,821 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:16:57,821 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. files: [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/bcc9230d27b14b59b579b86057f85d6f, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/6a9376887ebb42bd82b58762ca9ba345, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/0690eb4c09d04350b1ca995efd66b193] 2024-11-10T03:16:57,823 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting d2e10726678f411d91cfd9d550a14209, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731208614806 2024-11-10T03:16:57,823 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting bcc9230d27b14b59b579b86057f85d6f, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731208614806 2024-11-10T03:16:57,823 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 12332906f4fa4559ba1d1dcb7d756d47, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731208614945 2024-11-10T03:16:57,824 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 728939e45b6447189c598546d97266ba, keycount=350, bloomtype=ROW, size=21.5 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1731208617159 2024-11-10T03:16:57,824 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a9376887ebb42bd82b58762ca9ba345, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731208614945 2024-11-10T03:16:57,826 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0690eb4c09d04350b1ca995efd66b193, keycount=350, bloomtype=ROW, size=64.5 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1731208617159 2024-11-10T03:16:57,841 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f294ea12149d12df042ace2dfb55c011] 2024-11-10T03:16:57,842 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f294ea12149d12df042ace2dfb55c011#B#compaction#165 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:16:57,843 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/B/22cbab8b0d5d42f1985065431b1780f8 is 50, key is test_row_0/B:col10/1731208617187/Put/seqid=0 2024-11-10T03:16:57,846 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241110ead91ac3d7894ffc8451d7aab96a64ae_f294ea12149d12df042ace2dfb55c011 store=[table=TestAcidGuarantees family=A region=f294ea12149d12df042ace2dfb55c011] 2024-11-10T03:16:57,849 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241110ead91ac3d7894ffc8451d7aab96a64ae_f294ea12149d12df042ace2dfb55c011, store=[table=TestAcidGuarantees family=A region=f294ea12149d12df042ace2dfb55c011] 2024-11-10T03:16:57,849 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110ead91ac3d7894ffc8451d7aab96a64ae_f294ea12149d12df042ace2dfb55c011 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f294ea12149d12df042ace2dfb55c011] 2024-11-10T03:16:57,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742026_1202 (size=13017) 2024-11-10T03:16:57,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742027_1203 (size=4469) 2024-11-10T03:16:57,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-10T03:16:57,956 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T03:16:57,956 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T03:16:57,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-10T03:16:57,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-10T03:16:57,962 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T03:16:57,964 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T03:16:57,964 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T03:16:58,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on f294ea12149d12df042ace2dfb55c011 2024-11-10T03:16:58,027 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f294ea12149d12df042ace2dfb55c011 3/3 column families, dataSize=147.60 KB heapSize=420.84 KB 2024-11-10T03:16:58,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f294ea12149d12df042ace2dfb55c011, store=A 2024-11-10T03:16:58,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-10T03:16:58,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f294ea12149d12df042ace2dfb55c011, store=B 2024-11-10T03:16:58,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-10T03:16:58,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f294ea12149d12df042ace2dfb55c011, store=C 2024-11-10T03:16:58,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-10T03:16:58,037 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:58,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37896 deadline: 1731208678035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:58,038 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:58,038 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:58,038 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:58,038 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:58,038 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:58,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37968 deadline: 1731208678038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:58,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37958 deadline: 1731208678038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:58,038 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:58,038 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:58,038 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:58,039 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:58,039 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:58,039 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:58,042 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110b09b3ced056a4018bb2bdc812a879e60_f294ea12149d12df042ace2dfb55c011 is 50, key is test_row_0/A:col10/1731208617372/Put/seqid=0 2024-11-10T03:16:58,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742028_1204 (size=14994) 2024-11-10T03:16:58,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-10T03:16:58,120 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-10T03:16:58,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:16:58,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. as already flushing 2024-11-10T03:16:58,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:16:58,121 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:58,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:58,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:58,148 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:58,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37896 deadline: 1731208678142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:58,148 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:58,149 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:58,149 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:58,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37958 deadline: 1731208678143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:58,149 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:58,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:58,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:58,149 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:58,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:58,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37968 deadline: 1731208678145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:58,149 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:58,149 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:58,149 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:58,256 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/B/22cbab8b0d5d42f1985065431b1780f8 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/22cbab8b0d5d42f1985065431b1780f8 2024-11-10T03:16:58,259 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f294ea12149d12df042ace2dfb55c011#A#compaction#166 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:16:58,259 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/A/1ae3dc7a46cb415e8ae950737750eb55 is 175, key is test_row_0/A:col10/1731208617187/Put/seqid=0 2024-11-10T03:16:58,262 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f294ea12149d12df042ace2dfb55c011/B of f294ea12149d12df042ace2dfb55c011 into 22cbab8b0d5d42f1985065431b1780f8(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:16:58,262 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f294ea12149d12df042ace2dfb55c011: 2024-11-10T03:16:58,262 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., storeName=f294ea12149d12df042ace2dfb55c011/B, priority=13, startTime=1731208617817; duration=0sec 2024-11-10T03:16:58,262 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T03:16:58,263 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f294ea12149d12df042ace2dfb55c011:B 2024-11-10T03:16:58,263 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T03:16:58,263 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T03:16:58,264 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1541): f294ea12149d12df042ace2dfb55c011/C is initiating minor compaction (all files) 2024-11-10T03:16:58,264 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f294ea12149d12df042ace2dfb55c011/C in TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:16:58,264 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/8a690cb9d0f84df4ac4d2c37e75893ef, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/5bc9d89143594d0bb42e955ec7d8114c, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/abd12603d8a944bbac02793dd577f871] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp, totalSize=36.6 K 2024-11-10T03:16:58,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742029_1205 (size=31971) 2024-11-10T03:16:58,265 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a690cb9d0f84df4ac4d2c37e75893ef, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731208614806 2024-11-10T03:16:58,265 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 5bc9d89143594d0bb42e955ec7d8114c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731208614945 2024-11-10T03:16:58,265 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting abd12603d8a944bbac02793dd577f871, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1731208617161 2024-11-10T03:16:58,270 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/A/1ae3dc7a46cb415e8ae950737750eb55 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/1ae3dc7a46cb415e8ae950737750eb55 2024-11-10T03:16:58,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-10T03:16:58,277 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-10T03:16:58,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:16:58,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. as already flushing 2024-11-10T03:16:58,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:16:58,277 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:58,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:58,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:58,279 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f294ea12149d12df042ace2dfb55c011/A of f294ea12149d12df042ace2dfb55c011 into 1ae3dc7a46cb415e8ae950737750eb55(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:16:58,279 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f294ea12149d12df042ace2dfb55c011: 2024-11-10T03:16:58,279 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., storeName=f294ea12149d12df042ace2dfb55c011/A, priority=13, startTime=1731208617817; duration=0sec 2024-11-10T03:16:58,280 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:16:58,280 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f294ea12149d12df042ace2dfb55c011:A 2024-11-10T03:16:58,290 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f294ea12149d12df042ace2dfb55c011#C#compaction#168 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:16:58,290 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/C/07b7a6e240b24047ac0b5638ba5e4894 is 50, key is test_row_0/C:col10/1731208617187/Put/seqid=0 2024-11-10T03:16:58,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742030_1206 (size=13017) 2024-11-10T03:16:58,316 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/C/07b7a6e240b24047ac0b5638ba5e4894 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/07b7a6e240b24047ac0b5638ba5e4894 2024-11-10T03:16:58,324 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f294ea12149d12df042ace2dfb55c011/C of f294ea12149d12df042ace2dfb55c011 into 07b7a6e240b24047ac0b5638ba5e4894(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:16:58,324 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f294ea12149d12df042ace2dfb55c011: 2024-11-10T03:16:58,324 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., storeName=f294ea12149d12df042ace2dfb55c011/C, priority=13, startTime=1731208617817; duration=0sec 2024-11-10T03:16:58,325 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:16:58,325 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f294ea12149d12df042ace2dfb55c011:C 2024-11-10T03:16:58,358 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:58,358 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:58,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37896 deadline: 1731208678357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:58,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37968 deadline: 1731208678357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:58,358 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:58,359 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:58,359 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:58,359 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:58,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37958 deadline: 1731208678357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:58,360 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:58,360 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:58,360 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:58,360 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:58,360 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:58,360 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:58,431 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-10T03:16:58,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:16:58,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. as already flushing 2024-11-10T03:16:58,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:16:58,431 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:58,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:58,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:58,455 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:58,459 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110b09b3ced056a4018bb2bdc812a879e60_f294ea12149d12df042ace2dfb55c011 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110b09b3ced056a4018bb2bdc812a879e60_f294ea12149d12df042ace2dfb55c011 2024-11-10T03:16:58,460 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/A/632e920b49b94953bb1a920b261ca240, store: [table=TestAcidGuarantees family=A region=f294ea12149d12df042ace2dfb55c011] 2024-11-10T03:16:58,460 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/A/632e920b49b94953bb1a920b261ca240 is 175, key is test_row_0/A:col10/1731208617372/Put/seqid=0 2024-11-10T03:16:58,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742031_1207 (size=39949) 2024-11-10T03:16:58,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-10T03:16:58,584 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-10T03:16:58,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:16:58,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. as already flushing 2024-11-10T03:16:58,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:16:58,584 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:58,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:58,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:58,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:58,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37896 deadline: 1731208678663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:58,664 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:58,664 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:58,664 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:58,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:58,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37958 deadline: 1731208678676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:58,684 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:58,684 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:58,684 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:58,684 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:58,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37968 deadline: 1731208678676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:58,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:58,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:58,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:58,738 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-10T03:16:58,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:16:58,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. as already flushing 2024-11-10T03:16:58,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:16:58,745 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:58,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:58,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:58,874 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=337, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/A/632e920b49b94953bb1a920b261ca240 2024-11-10T03:16:58,895 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/B/7db358bd4959452b8c8cc6ce96409b32 is 50, key is test_row_0/B:col10/1731208617372/Put/seqid=0 2024-11-10T03:16:58,909 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-10T03:16:58,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:16:58,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. as already flushing 2024-11-10T03:16:58,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:16:58,913 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:58,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:58,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:16:58,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742032_1208 (size=12301) 2024-11-10T03:16:58,931 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/B/7db358bd4959452b8c8cc6ce96409b32 2024-11-10T03:16:58,959 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/C/90738642720c422f8bca2f33db574a2a is 50, key is test_row_0/C:col10/1731208617372/Put/seqid=0 2024-11-10T03:16:58,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742033_1209 (size=12301) 2024-11-10T03:16:58,998 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/C/90738642720c422f8bca2f33db574a2a 2024-11-10T03:16:59,022 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/A/632e920b49b94953bb1a920b261ca240 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/632e920b49b94953bb1a920b261ca240 2024-11-10T03:16:59,026 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/632e920b49b94953bb1a920b261ca240, entries=200, sequenceid=337, filesize=39.0 K 2024-11-10T03:16:59,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,028 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/B/7db358bd4959452b8c8cc6ce96409b32 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/7db358bd4959452b8c8cc6ce96409b32 2024-11-10T03:16:59,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,032 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/7db358bd4959452b8c8cc6ce96409b32, entries=150, sequenceid=337, filesize=12.0 K 2024-11-10T03:16:59,033 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/C/90738642720c422f8bca2f33db574a2a as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/90738642720c422f8bca2f33db574a2a 2024-11-10T03:16:59,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,038 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/90738642720c422f8bca2f33db574a2a, entries=150, sequenceid=337, filesize=12.0 K 2024-11-10T03:16:59,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,039 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~420.80 KB/430896, currentSize=33.54 KB/34350 for f294ea12149d12df042ace2dfb55c011 in 1012ms, sequenceid=337, compaction requested=false 2024-11-10T03:16:59,039 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f294ea12149d12df042ace2dfb55c011: 2024-11-10T03:16:59,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,066 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-10T03:16:59,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:16:59,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,066 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing f294ea12149d12df042ace2dfb55c011 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-10T03:16:59,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f294ea12149d12df042ace2dfb55c011, store=A 2024-11-10T03:16:59,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:16:59,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f294ea12149d12df042ace2dfb55c011, store=B 2024-11-10T03:16:59,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:16:59,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f294ea12149d12df042ace2dfb55c011, store=C 2024-11-10T03:16:59,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:16:59,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110e28a92fe8395480f8df86eac5dd9713f_f294ea12149d12df042ace2dfb55c011 is 50, key is test_row_1/A:col10/1731208618036/Put/seqid=0 2024-11-10T03:16:59,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-10T03:16:59,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742034_1210 (size=9914) 2024-11-10T03:16:59,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,138 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110e28a92fe8395480f8df86eac5dd9713f_f294ea12149d12df042ace2dfb55c011 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110e28a92fe8395480f8df86eac5dd9713f_f294ea12149d12df042ace2dfb55c011 2024-11-10T03:16:59,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/A/4bc11ee82f3b4306a8e1ef0cc0c5b164, store: [table=TestAcidGuarantees family=A region=f294ea12149d12df042ace2dfb55c011] 2024-11-10T03:16:59,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/A/4bc11ee82f3b4306a8e1ef0cc0c5b164 is 175, key is test_row_1/A:col10/1731208618036/Put/seqid=0 2024-11-10T03:16:59,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742035_1211 (size=22561) 2024-11-10T03:16:59,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,167 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=348, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/A/4bc11ee82f3b4306a8e1ef0cc0c5b164 2024-11-10T03:16:59,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/B/863e68d63eca4b52928535415f3b5da2 is 50, key is test_row_1/B:col10/1731208618036/Put/seqid=0 2024-11-10T03:16:59,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:16:59,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on f294ea12149d12df042ace2dfb55c011 2024-11-10T03:16:59,198 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. as already flushing 2024-11-10T03:16:59,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742036_1212 (size=9857) 2024-11-10T03:16:59,228 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/B/863e68d63eca4b52928535415f3b5da2 2024-11-10T03:16:59,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/C/1457d735e0cf45fabc9f4b611448690e is 50, key is test_row_1/C:col10/1731208618036/Put/seqid=0 2024-11-10T03:16:59,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742037_1213 (size=9857) 2024-11-10T03:16:59,245 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731208619221, max timestamp=1731208619242Num uniques 859; 2024-11-10T03:16:59,245 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731208619221, max timestamp=1731208619242Num uniques 859; 2024-11-10T03:16:59,245 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-10T03:16:59,245 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-10T03:16:59,245 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731208619221, max timestamp=1731208619242Num uniques 859; 2024-11-10T03:16:59,245 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-10T03:16:59,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:59,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37958 deadline: 1731208679269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:59,278 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:59,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37896 deadline: 1731208679272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:59,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:59,279 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:59,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:59,279 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:59,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:59,279 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:59,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:59,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37968 deadline: 1731208679278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:59,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:59,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:59,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:59,383 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:59,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37958 deadline: 1731208679382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:59,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:59,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37968 deadline: 1731208679383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:59,384 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:59,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:59,384 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:59,384 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:59,384 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:59,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37896 deadline: 1731208679384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:59,384 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:59,384 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:59,384 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:59,384 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:59,384 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:59,544 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:59,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37914 deadline: 1731208679543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:59,545 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 394d2222f724,38139,1731208572054 for 'test_row_0' in f294ea12149d12df042ace2dfb55c011 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T03:16:59,546 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:59,546 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:59,546 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:59,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:59,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37948 deadline: 1731208679572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:59,580 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 394d2222f724,38139,1731208572054 for 'test_row_1' in f294ea12149d12df042ace2dfb55c011 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8231 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T03:16:59,581 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:59,581 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:59,581 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:59,595 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:59,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37958 deadline: 1731208679593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:59,596 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:59,596 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:59,596 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:59,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37968 deadline: 1731208679593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:59,596 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:59,596 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:59,596 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:59,596 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:59,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37896 deadline: 1731208679593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:59,596 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:59,596 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:59,596 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:59,596 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:59,642 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/C/1457d735e0cf45fabc9f4b611448690e 2024-11-10T03:16:59,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/A/4bc11ee82f3b4306a8e1ef0cc0c5b164 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/4bc11ee82f3b4306a8e1ef0cc0c5b164 2024-11-10T03:16:59,651 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/4bc11ee82f3b4306a8e1ef0cc0c5b164, entries=100, sequenceid=348, filesize=22.0 K 2024-11-10T03:16:59,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/B/863e68d63eca4b52928535415f3b5da2 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/863e68d63eca4b52928535415f3b5da2 2024-11-10T03:16:59,656 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/863e68d63eca4b52928535415f3b5da2, entries=100, sequenceid=348, filesize=9.6 K 2024-11-10T03:16:59,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/C/1457d735e0cf45fabc9f4b611448690e as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/1457d735e0cf45fabc9f4b611448690e 2024-11-10T03:16:59,662 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/1457d735e0cf45fabc9f4b611448690e, entries=100, sequenceid=348, filesize=9.6 K 2024-11-10T03:16:59,663 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=147.60 KB/151140 for f294ea12149d12df042ace2dfb55c011 in 597ms, sequenceid=348, compaction requested=true 2024-11-10T03:16:59,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for f294ea12149d12df042ace2dfb55c011: 2024-11-10T03:16:59,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:16:59,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-10T03:16:59,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-10T03:16:59,666 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-10T03:16:59,666 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7010 sec 2024-11-10T03:16:59,668 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 1.7110 sec 2024-11-10T03:16:59,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on f294ea12149d12df042ace2dfb55c011 2024-11-10T03:16:59,904 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f294ea12149d12df042ace2dfb55c011 3/3 column families, dataSize=154.31 KB heapSize=436.08 KB 2024-11-10T03:16:59,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f294ea12149d12df042ace2dfb55c011, store=A 2024-11-10T03:16:59,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-10T03:16:59,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f294ea12149d12df042ace2dfb55c011, store=B 2024-11-10T03:16:59,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-10T03:16:59,905 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f294ea12149d12df042ace2dfb55c011, store=C 2024-11-10T03:16:59,905 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-10T03:16:59,913 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:59,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37896 deadline: 1731208679907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:59,913 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:59,913 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:59,913 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:59,915 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:59,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37958 deadline: 1731208679908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:59,915 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:59,915 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:59,915 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:59,917 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110a95f48bdd687472ebfa049edb22a5413_f294ea12149d12df042ace2dfb55c011 is 50, key is test_row_0/A:col10/1731208619270/Put/seqid=0 2024-11-10T03:16:59,919 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:16:59,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37968 deadline: 1731208679913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:16:59,920 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:59,920 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:16:59,920 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:16:59,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742038_1214 (size=14994) 2024-11-10T03:17:00,025 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:00,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:00,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37896 deadline: 1731208680023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:00,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37958 deadline: 1731208680023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:00,025 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:00,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:00,025 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:00,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37968 deadline: 1731208680023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:00,026 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:00,026 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:00,026 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:00,026 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:00,026 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:00,026 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:00,026 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:00,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-10T03:17:00,103 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T03:17:00,103 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T03:17:00,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-10T03:17:00,105 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T03:17:00,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-10T03:17:00,106 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T03:17:00,106 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T03:17:00,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-10T03:17:00,235 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:00,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37958 deadline: 1731208680233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:00,236 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:00,236 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:00,236 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:00,236 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:00,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37968 deadline: 1731208680233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:00,236 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:00,236 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:00,236 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:00,236 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:00,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37896 deadline: 1731208680233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:00,237 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:00,237 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:00,237 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:00,258 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-10T03:17:00,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:17:00,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. as already flushing 2024-11-10T03:17:00,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:17:00,259 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:00,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:00,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:00,322 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:00,326 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110a95f48bdd687472ebfa049edb22a5413_f294ea12149d12df042ace2dfb55c011 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110a95f48bdd687472ebfa049edb22a5413_f294ea12149d12df042ace2dfb55c011 2024-11-10T03:17:00,327 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/A/311f5bb07518487a94dc40e1df08b0a4, store: [table=TestAcidGuarantees family=A region=f294ea12149d12df042ace2dfb55c011] 2024-11-10T03:17:00,328 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/A/311f5bb07518487a94dc40e1df08b0a4 is 175, key is test_row_0/A:col10/1731208619270/Put/seqid=0 2024-11-10T03:17:00,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742039_1215 (size=39949) 2024-11-10T03:17:00,412 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-10T03:17:00,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:17:00,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. as already flushing 2024-11-10T03:17:00,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:17:00,413 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:00,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:00,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:00,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-10T03:17:00,549 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:00,549 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:00,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37968 deadline: 1731208680543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:00,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37958 deadline: 1731208680543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:00,549 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:00,549 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:00,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:00,549 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:00,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37896 deadline: 1731208680543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:00,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:00,549 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:00,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:00,549 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:00,549 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:00,549 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:00,565 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-10T03:17:00,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:17:00,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. as already flushing 2024-11-10T03:17:00,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:17:00,566 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:00,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:00,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:00,718 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-10T03:17:00,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:17:00,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. as already flushing 2024-11-10T03:17:00,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:17:00,718 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:00,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:00,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:00,732 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=386, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/A/311f5bb07518487a94dc40e1df08b0a4 2024-11-10T03:17:00,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-10T03:17:00,747 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/B/9e19e1cf342a461ba6ca3529bd03b6db is 50, key is test_row_0/B:col10/1731208619270/Put/seqid=0 2024-11-10T03:17:00,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742040_1216 (size=12301) 2024-11-10T03:17:00,756 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=386 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/B/9e19e1cf342a461ba6ca3529bd03b6db 2024-11-10T03:17:00,767 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/C/43e81a29b3a14879ae1e8ef35a2964d1 is 50, key is test_row_0/C:col10/1731208619270/Put/seqid=0 2024-11-10T03:17:00,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742041_1217 (size=12301) 2024-11-10T03:17:00,871 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-10T03:17:00,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:17:00,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. as already flushing 2024-11-10T03:17:00,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:17:00,871 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:00,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:00,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:01,026 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-10T03:17:01,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:17:01,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. as already flushing 2024-11-10T03:17:01,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:17:01,026 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:01,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:01,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:01,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:01,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37896 deadline: 1731208681053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:01,055 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:01,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37968 deadline: 1731208681053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:01,055 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:01,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:01,055 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:01,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:01,055 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:01,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:01,068 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:01,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37958 deadline: 1731208681063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:01,068 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:01,068 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:01,068 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:01,179 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-10T03:17:01,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:17:01,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. as already flushing 2024-11-10T03:17:01,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:17:01,179 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:01,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:01,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:01,181 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=386 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/C/43e81a29b3a14879ae1e8ef35a2964d1 2024-11-10T03:17:01,192 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/A/311f5bb07518487a94dc40e1df08b0a4 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/311f5bb07518487a94dc40e1df08b0a4 2024-11-10T03:17:01,200 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/311f5bb07518487a94dc40e1df08b0a4, entries=200, sequenceid=386, filesize=39.0 K 2024-11-10T03:17:01,202 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/B/9e19e1cf342a461ba6ca3529bd03b6db as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/9e19e1cf342a461ba6ca3529bd03b6db 2024-11-10T03:17:01,209 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/9e19e1cf342a461ba6ca3529bd03b6db, entries=150, sequenceid=386, filesize=12.0 K 2024-11-10T03:17:01,210 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/C/43e81a29b3a14879ae1e8ef35a2964d1 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/43e81a29b3a14879ae1e8ef35a2964d1 2024-11-10T03:17:01,218 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/43e81a29b3a14879ae1e8ef35a2964d1, entries=150, sequenceid=386, filesize=12.0 K 2024-11-10T03:17:01,219 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~436.03 KB/446496, currentSize=26.84 KB/27480 for f294ea12149d12df042ace2dfb55c011 in 1315ms, sequenceid=386, compaction requested=true 2024-11-10T03:17:01,219 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f294ea12149d12df042ace2dfb55c011: 2024-11-10T03:17:01,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f294ea12149d12df042ace2dfb55c011:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T03:17:01,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:17:01,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f294ea12149d12df042ace2dfb55c011:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T03:17:01,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:17:01,219 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T03:17:01,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f294ea12149d12df042ace2dfb55c011:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T03:17:01,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T03:17:01,219 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T03:17:01,222 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47476 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T03:17:01,222 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1541): f294ea12149d12df042ace2dfb55c011/B is initiating minor compaction (all files) 2024-11-10T03:17:01,222 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f294ea12149d12df042ace2dfb55c011/B in TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:17:01,222 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/22cbab8b0d5d42f1985065431b1780f8, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/7db358bd4959452b8c8cc6ce96409b32, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/863e68d63eca4b52928535415f3b5da2, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/9e19e1cf342a461ba6ca3529bd03b6db] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp, totalSize=46.4 K 2024-11-10T03:17:01,223 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 22cbab8b0d5d42f1985065431b1780f8, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1731208617161 2024-11-10T03:17:01,223 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 134430 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T03:17:01,223 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1541): f294ea12149d12df042ace2dfb55c011/A is initiating minor compaction (all files) 2024-11-10T03:17:01,223 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f294ea12149d12df042ace2dfb55c011/A in TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:17:01,223 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/1ae3dc7a46cb415e8ae950737750eb55, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/632e920b49b94953bb1a920b261ca240, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/4bc11ee82f3b4306a8e1ef0cc0c5b164, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/311f5bb07518487a94dc40e1df08b0a4] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp, totalSize=131.3 K 2024-11-10T03:17:01,223 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:17:01,223 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. files: [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/1ae3dc7a46cb415e8ae950737750eb55, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/632e920b49b94953bb1a920b261ca240, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/4bc11ee82f3b4306a8e1ef0cc0c5b164, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/311f5bb07518487a94dc40e1df08b0a4] 2024-11-10T03:17:01,224 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 7db358bd4959452b8c8cc6ce96409b32, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1731208617372 2024-11-10T03:17:01,225 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ae3dc7a46cb415e8ae950737750eb55, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1731208617161 2024-11-10T03:17:01,226 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 863e68d63eca4b52928535415f3b5da2, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1731208618034 2024-11-10T03:17:01,226 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 632e920b49b94953bb1a920b261ca240, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1731208617371 2024-11-10T03:17:01,226 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e19e1cf342a461ba6ca3529bd03b6db, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=386, earliestPutTs=1731208619270 2024-11-10T03:17:01,227 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4bc11ee82f3b4306a8e1ef0cc0c5b164, keycount=100, bloomtype=ROW, size=22.0 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1731208618034 2024-11-10T03:17:01,227 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 311f5bb07518487a94dc40e1df08b0a4, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=386, earliestPutTs=1731208619268 2024-11-10T03:17:01,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-10T03:17:01,248 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f294ea12149d12df042ace2dfb55c011] 2024-11-10T03:17:01,252 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f294ea12149d12df042ace2dfb55c011#B#compaction#178 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:17:01,252 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/B/54f360ed8c6b487e866b89aa402a0978 is 50, key is test_row_0/B:col10/1731208619270/Put/seqid=0 2024-11-10T03:17:01,265 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411107512c5d2dc0442999ead8fe9b2c675cb_f294ea12149d12df042ace2dfb55c011 store=[table=TestAcidGuarantees family=A region=f294ea12149d12df042ace2dfb55c011] 2024-11-10T03:17:01,267 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411107512c5d2dc0442999ead8fe9b2c675cb_f294ea12149d12df042ace2dfb55c011, store=[table=TestAcidGuarantees family=A region=f294ea12149d12df042ace2dfb55c011] 2024-11-10T03:17:01,268 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411107512c5d2dc0442999ead8fe9b2c675cb_f294ea12149d12df042ace2dfb55c011 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f294ea12149d12df042ace2dfb55c011] 2024-11-10T03:17:01,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742042_1218 (size=13153) 2024-11-10T03:17:01,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742043_1219 (size=4469) 2024-11-10T03:17:01,332 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-10T03:17:01,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:17:01,332 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing f294ea12149d12df042ace2dfb55c011 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-10T03:17:01,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f294ea12149d12df042ace2dfb55c011, store=A 2024-11-10T03:17:01,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:17:01,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f294ea12149d12df042ace2dfb55c011, store=B 2024-11-10T03:17:01,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:17:01,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f294ea12149d12df042ace2dfb55c011, store=C 2024-11-10T03:17:01,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:17:01,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110408948ca3cfc4b54bfdcff4914f8901c_f294ea12149d12df042ace2dfb55c011 is 50, key is test_row_0/A:col10/1731208619907/Put/seqid=0 2024-11-10T03:17:01,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742044_1220 (size=12454) 2024-11-10T03:17:01,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:01,350 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110408948ca3cfc4b54bfdcff4914f8901c_f294ea12149d12df042ace2dfb55c011 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110408948ca3cfc4b54bfdcff4914f8901c_f294ea12149d12df042ace2dfb55c011 2024-11-10T03:17:01,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/A/dca4c9d025e84bda86f62d4ef16a31cc, store: [table=TestAcidGuarantees family=A region=f294ea12149d12df042ace2dfb55c011] 2024-11-10T03:17:01,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/A/dca4c9d025e84bda86f62d4ef16a31cc is 175, key is test_row_0/A:col10/1731208619907/Put/seqid=0 2024-11-10T03:17:01,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742045_1221 (size=31255) 2024-11-10T03:17:01,694 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f294ea12149d12df042ace2dfb55c011#A#compaction#177 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:17:01,694 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/A/a38211a9e93d4e659d90f07e66879b4e is 175, key is test_row_0/A:col10/1731208619270/Put/seqid=0 2024-11-10T03:17:01,696 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/B/54f360ed8c6b487e866b89aa402a0978 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/54f360ed8c6b487e866b89aa402a0978 2024-11-10T03:17:01,702 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f294ea12149d12df042ace2dfb55c011/B of f294ea12149d12df042ace2dfb55c011 into 54f360ed8c6b487e866b89aa402a0978(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:17:01,702 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f294ea12149d12df042ace2dfb55c011: 2024-11-10T03:17:01,702 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., storeName=f294ea12149d12df042ace2dfb55c011/B, priority=12, startTime=1731208621219; duration=0sec 2024-11-10T03:17:01,702 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T03:17:01,703 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f294ea12149d12df042ace2dfb55c011:B 2024-11-10T03:17:01,703 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T03:17:01,704 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47476 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T03:17:01,704 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1541): f294ea12149d12df042ace2dfb55c011/C is initiating minor compaction (all files) 2024-11-10T03:17:01,704 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f294ea12149d12df042ace2dfb55c011/C in TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:17:01,704 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/07b7a6e240b24047ac0b5638ba5e4894, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/90738642720c422f8bca2f33db574a2a, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/1457d735e0cf45fabc9f4b611448690e, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/43e81a29b3a14879ae1e8ef35a2964d1] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp, totalSize=46.4 K 2024-11-10T03:17:01,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742046_1222 (size=32107) 2024-11-10T03:17:01,704 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 07b7a6e240b24047ac0b5638ba5e4894, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1731208617161 2024-11-10T03:17:01,705 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 90738642720c422f8bca2f33db574a2a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1731208617372 2024-11-10T03:17:01,705 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 1457d735e0cf45fabc9f4b611448690e, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1731208618034 2024-11-10T03:17:01,706 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 43e81a29b3a14879ae1e8ef35a2964d1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=386, earliestPutTs=1731208619270 2024-11-10T03:17:01,715 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f294ea12149d12df042ace2dfb55c011#C#compaction#180 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:17:01,716 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/C/d5a8ec1471ca423d952573202a16edca is 50, key is test_row_0/C:col10/1731208619270/Put/seqid=0 2024-11-10T03:17:01,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742047_1223 (size=13153) 2024-11-10T03:17:01,758 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=393, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/A/dca4c9d025e84bda86f62d4ef16a31cc 2024-11-10T03:17:01,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/B/3c053d4d3fff468aaa4dfea08976ee51 is 50, key is test_row_0/B:col10/1731208619907/Put/seqid=0 2024-11-10T03:17:01,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742048_1224 (size=12301) 2024-11-10T03:17:01,785 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/B/3c053d4d3fff468aaa4dfea08976ee51 2024-11-10T03:17:01,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/C/00d998eb554240b1bfaaa07b002c6c8b is 50, key is test_row_0/C:col10/1731208619907/Put/seqid=0 2024-11-10T03:17:01,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742049_1225 (size=12301) 2024-11-10T03:17:02,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on f294ea12149d12df042ace2dfb55c011 2024-11-10T03:17:02,077 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. as already flushing 2024-11-10T03:17:02,106 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731208622088, max timestamp=1731208622103Num uniques 859; 2024-11-10T03:17:02,106 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-10T03:17:02,108 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731208622088, max timestamp=1731208622103Num uniques 859; 2024-11-10T03:17:02,108 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-10T03:17:02,110 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731208622088, max timestamp=1731208622103Num uniques 859; 2024-11-10T03:17:02,110 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-10T03:17:02,111 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/A/a38211a9e93d4e659d90f07e66879b4e as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/a38211a9e93d4e659d90f07e66879b4e 2024-11-10T03:17:02,119 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f294ea12149d12df042ace2dfb55c011/A of f294ea12149d12df042ace2dfb55c011 into a38211a9e93d4e659d90f07e66879b4e(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:17:02,119 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f294ea12149d12df042ace2dfb55c011: 2024-11-10T03:17:02,119 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., storeName=f294ea12149d12df042ace2dfb55c011/A, priority=12, startTime=1731208621219; duration=0sec 2024-11-10T03:17:02,120 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:17:02,120 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f294ea12149d12df042ace2dfb55c011:A 2024-11-10T03:17:02,149 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/C/d5a8ec1471ca423d952573202a16edca as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/d5a8ec1471ca423d952573202a16edca 2024-11-10T03:17:02,161 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f294ea12149d12df042ace2dfb55c011/C of f294ea12149d12df042ace2dfb55c011 into d5a8ec1471ca423d952573202a16edca(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:17:02,161 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f294ea12149d12df042ace2dfb55c011: 2024-11-10T03:17:02,161 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., storeName=f294ea12149d12df042ace2dfb55c011/C, priority=12, startTime=1731208621219; duration=0sec 2024-11-10T03:17:02,161 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:17:02,161 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f294ea12149d12df042ace2dfb55c011:C 2024-11-10T03:17:02,195 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731208622159, max timestamp=1731208622191Num uniques 1118; 2024-11-10T03:17:02,196 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731208622159, max timestamp=1731208622191Num uniques 1118; 2024-11-10T03:17:02,196 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=12268, suffix heap size=144240,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=268 2024-11-10T03:17:02,196 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=12268, suffix heap size=144240,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=268 2024-11-10T03:17:02,198 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731208622159, max timestamp=1731208622191Num uniques 1009; 2024-11-10T03:17:02,198 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46213, new segment data size=12268, suffix heap size=132904,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=268 2024-11-10T03:17:02,207 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/C/00d998eb554240b1bfaaa07b002c6c8b 2024-11-10T03:17:02,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/A/dca4c9d025e84bda86f62d4ef16a31cc as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/dca4c9d025e84bda86f62d4ef16a31cc 2024-11-10T03:17:02,222 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/dca4c9d025e84bda86f62d4ef16a31cc, entries=150, sequenceid=393, filesize=30.5 K 2024-11-10T03:17:02,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/B/3c053d4d3fff468aaa4dfea08976ee51 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/3c053d4d3fff468aaa4dfea08976ee51 2024-11-10T03:17:02,229 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/3c053d4d3fff468aaa4dfea08976ee51, entries=150, sequenceid=393, filesize=12.0 K 2024-11-10T03:17:02,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/C/00d998eb554240b1bfaaa07b002c6c8b as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/00d998eb554240b1bfaaa07b002c6c8b 2024-11-10T03:17:02,235 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/00d998eb554240b1bfaaa07b002c6c8b, entries=150, sequenceid=393, filesize=12.0 K 2024-11-10T03:17:02,236 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=114.05 KB/116790 for f294ea12149d12df042ace2dfb55c011 in 904ms, sequenceid=393, compaction requested=false 2024-11-10T03:17:02,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for f294ea12149d12df042ace2dfb55c011: 2024-11-10T03:17:02,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:17:02,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-10T03:17:02,237 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f294ea12149d12df042ace2dfb55c011 3/3 column families, dataSize=120.76 KB heapSize=334.92 KB 2024-11-10T03:17:02,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f294ea12149d12df042ace2dfb55c011, store=A 2024-11-10T03:17:02,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-10T03:17:02,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T03:17:02,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f294ea12149d12df042ace2dfb55c011, store=B 2024-11-10T03:17:02,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T03:17:02,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f294ea12149d12df042ace2dfb55c011, store=C 2024-11-10T03:17:02,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T03:17:02,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on f294ea12149d12df042ace2dfb55c011 2024-11-10T03:17:02,246 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-10T03:17:02,246 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1320 sec 2024-11-10T03:17:02,248 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 2.1430 sec 2024-11-10T03:17:02,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-10T03:17:02,254 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T03:17:02,254 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T03:17:02,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-10T03:17:02,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-10T03:17:02,257 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T03:17:02,258 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T03:17:02,258 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T03:17:02,270 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110546774c3de724b8083d2966ef29818fc_f294ea12149d12df042ace2dfb55c011 is 50, key is test_row_0/A:col10/1731208622201/Put/seqid=0 2024-11-10T03:17:02,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742050_1226 (size=14994) 2024-11-10T03:17:02,304 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:02,305 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:02,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37958 deadline: 1731208682294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:02,305 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:02,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37896 deadline: 1731208682295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:02,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37968 deadline: 1731208682292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:02,306 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:02,306 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:02,306 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:02,306 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:02,306 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:02,306 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:02,307 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:02,307 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:02,307 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:02,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-10T03:17:02,413 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-10T03:17:02,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:17:02,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. as already flushing 2024-11-10T03:17:02,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:17:02,413 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:02,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:02,414 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:02,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37968 deadline: 1731208682413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:02,414 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:02,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:02,414 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:02,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37896 deadline: 1731208682413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:02,414 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:02,415 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:02,415 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:02,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37958 deadline: 1731208682413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:02,415 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:02,415 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:02,416 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:02,416 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:02,416 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:02,416 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:02,567 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-10T03:17:02,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:17:02,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. as already flushing 2024-11-10T03:17:02,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:17:02,567 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:02,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:02,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:02,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-10T03:17:02,624 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:02,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37968 deadline: 1731208682622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:02,625 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:02,625 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:02,625 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:02,625 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:02,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37958 deadline: 1731208682623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:02,625 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:02,625 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:02,625 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:02,625 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:02,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37896 deadline: 1731208682623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:02,626 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:02,626 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:02,626 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:02,679 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:02,682 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110546774c3de724b8083d2966ef29818fc_f294ea12149d12df042ace2dfb55c011 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110546774c3de724b8083d2966ef29818fc_f294ea12149d12df042ace2dfb55c011 2024-11-10T03:17:02,684 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/A/a00853f4a2a240bf882f1b82dca2fb40, store: [table=TestAcidGuarantees family=A region=f294ea12149d12df042ace2dfb55c011] 2024-11-10T03:17:02,684 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/A/a00853f4a2a240bf882f1b82dca2fb40 is 175, key is test_row_0/A:col10/1731208622201/Put/seqid=0 2024-11-10T03:17:02,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742051_1227 (size=39949) 2024-11-10T03:17:02,720 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-10T03:17:02,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:17:02,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. as already flushing 2024-11-10T03:17:02,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:17:02,721 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:02,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:02,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:02,874 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-10T03:17:02,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:17:02,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. as already flushing 2024-11-10T03:17:02,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:17:02,874 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:02,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:02,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:02,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-10T03:17:02,933 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:02,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37958 deadline: 1731208682933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:02,933 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:02,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37896 deadline: 1731208682933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:02,933 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:02,933 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:02,933 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:02,934 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:02,934 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:02,934 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:02,936 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:02,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37968 deadline: 1731208682936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:02,937 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:02,937 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:02,937 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:03,022 INFO [Thread-615 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-615. 2024-11-10T03:17:03,022 DEBUG [Thread-615 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T03:17:03,022 DEBUG [Thread-615 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:17:03,023 DEBUG [Thread-615 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:17:03,023 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T03:17:03,025 INFO [Thread-609 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-609. 2024-11-10T03:17:03,025 DEBUG [Thread-609 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T03:17:03,025 DEBUG [Thread-609 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:17:03,025 DEBUG [Thread-609 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:17:03,025 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T03:17:03,028 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-10T03:17:03,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:17:03,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. as already flushing 2024-11-10T03:17:03,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:17:03,028 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:03,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:03,029 INFO [Thread-611 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-611. 2024-11-10T03:17:03,029 DEBUG [Thread-611 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T03:17:03,029 DEBUG [Thread-611 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:17:03,029 DEBUG [Thread-611 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:17:03,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:03,031 INFO [Thread-613 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-613. 2024-11-10T03:17:03,031 DEBUG [Thread-613 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T03:17:03,031 DEBUG [Thread-613 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:17:03,031 DEBUG [Thread-613 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:17:03,031 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T03:17:03,032 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T03:17:03,089 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=446, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/A/a00853f4a2a240bf882f1b82dca2fb40 2024-11-10T03:17:03,096 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/B/38b876ae322c44aea62d839bfcc42eb6 is 50, key is test_row_0/B:col10/1731208622201/Put/seqid=0 2024-11-10T03:17:03,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742052_1228 (size=12301) 2024-11-10T03:17:03,181 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-10T03:17:03,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:17:03,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. as already flushing 2024-11-10T03:17:03,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:17:03,182 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:03,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:03,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:03,334 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-10T03:17:03,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:17:03,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. as already flushing 2024-11-10T03:17:03,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:17:03,335 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:03,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:03,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:03,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-10T03:17:03,443 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:03,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:03,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37958 deadline: 1731208683443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:03,443 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:03,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37968 deadline: 1731208683443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:03,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37896 deadline: 1731208683443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:03,444 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:03,444 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:03,444 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:03,444 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:03,444 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:03,444 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:03,444 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:03,444 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:03,444 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:03,487 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-10T03:17:03,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:17:03,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. as already flushing 2024-11-10T03:17:03,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:17:03,488 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:03,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:03,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:03,502 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=446 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/B/38b876ae322c44aea62d839bfcc42eb6 2024-11-10T03:17:03,511 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/C/831f246d504d437e994397c3751e9fd0 is 50, key is test_row_0/C:col10/1731208622201/Put/seqid=0 2024-11-10T03:17:03,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742053_1229 (size=12301) 2024-11-10T03:17:03,641 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-10T03:17:03,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:17:03,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. as already flushing 2024-11-10T03:17:03,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:17:03,641 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:03,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:03,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:03,795 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-10T03:17:03,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:17:03,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. as already flushing 2024-11-10T03:17:03,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:17:03,795 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:03,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:03,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:03,915 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=446 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/C/831f246d504d437e994397c3751e9fd0 2024-11-10T03:17:03,923 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/A/a00853f4a2a240bf882f1b82dca2fb40 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/a00853f4a2a240bf882f1b82dca2fb40 2024-11-10T03:17:03,929 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/a00853f4a2a240bf882f1b82dca2fb40, entries=200, sequenceid=446, filesize=39.0 K 2024-11-10T03:17:03,932 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/B/38b876ae322c44aea62d839bfcc42eb6 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/38b876ae322c44aea62d839bfcc42eb6 2024-11-10T03:17:03,938 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/38b876ae322c44aea62d839bfcc42eb6, entries=150, sequenceid=446, filesize=12.0 K 2024-11-10T03:17:03,939 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/C/831f246d504d437e994397c3751e9fd0 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/831f246d504d437e994397c3751e9fd0 2024-11-10T03:17:03,945 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/831f246d504d437e994397c3751e9fd0, entries=150, sequenceid=446, filesize=12.0 K 2024-11-10T03:17:03,946 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~334.88 KB/342912, currentSize=60.38 KB/61830 for f294ea12149d12df042ace2dfb55c011 in 1709ms, sequenceid=446, compaction requested=true 2024-11-10T03:17:03,946 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f294ea12149d12df042ace2dfb55c011: 2024-11-10T03:17:03,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f294ea12149d12df042ace2dfb55c011:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T03:17:03,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:17:03,946 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T03:17:03,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f294ea12149d12df042ace2dfb55c011:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T03:17:03,947 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:17:03,947 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T03:17:03,947 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f294ea12149d12df042ace2dfb55c011:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T03:17:03,947 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T03:17:03,948 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-10T03:17:03,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:17:03,949 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T03:17:03,949 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1541): f294ea12149d12df042ace2dfb55c011/B is initiating minor compaction (all files) 2024-11-10T03:17:03,949 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f294ea12149d12df042ace2dfb55c011/B in TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:17:03,949 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/54f360ed8c6b487e866b89aa402a0978, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/3c053d4d3fff468aaa4dfea08976ee51, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/38b876ae322c44aea62d839bfcc42eb6] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp, totalSize=36.9 K 2024-11-10T03:17:03,950 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 54f360ed8c6b487e866b89aa402a0978, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=386, earliestPutTs=1731208619270 2024-11-10T03:17:03,951 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c053d4d3fff468aaa4dfea08976ee51, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1731208619906 2024-11-10T03:17:03,952 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 38b876ae322c44aea62d839bfcc42eb6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=446, earliestPutTs=1731208622201 2024-11-10T03:17:03,952 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing f294ea12149d12df042ace2dfb55c011 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-10T03:17:03,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f294ea12149d12df042ace2dfb55c011, store=A 2024-11-10T03:17:03,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:17:03,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f294ea12149d12df042ace2dfb55c011, store=B 2024-11-10T03:17:03,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:17:03,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f294ea12149d12df042ace2dfb55c011, store=C 2024-11-10T03:17:03,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:17:03,956 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103311 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T03:17:03,956 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1541): f294ea12149d12df042ace2dfb55c011/A is initiating minor compaction (all files) 2024-11-10T03:17:03,956 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f294ea12149d12df042ace2dfb55c011/A in TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:17:03,956 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/a38211a9e93d4e659d90f07e66879b4e, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/dca4c9d025e84bda86f62d4ef16a31cc, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/a00853f4a2a240bf882f1b82dca2fb40] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp, totalSize=100.9 K 2024-11-10T03:17:03,957 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:17:03,957 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. files: [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/a38211a9e93d4e659d90f07e66879b4e, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/dca4c9d025e84bda86f62d4ef16a31cc, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/a00853f4a2a240bf882f1b82dca2fb40] 2024-11-10T03:17:03,960 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting a38211a9e93d4e659d90f07e66879b4e, keycount=150, bloomtype=ROW, size=31.4 K, encoding=NONE, compression=NONE, seqNum=386, earliestPutTs=1731208619270 2024-11-10T03:17:03,961 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting dca4c9d025e84bda86f62d4ef16a31cc, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1731208619906 2024-11-10T03:17:03,962 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting a00853f4a2a240bf882f1b82dca2fb40, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=446, earliestPutTs=1731208622201 2024-11-10T03:17:03,966 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f294ea12149d12df042ace2dfb55c011#B#compaction#186 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:17:03,967 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/B/406abe1f75c440edbc3f2d37c77d944e is 50, key is test_row_0/B:col10/1731208622201/Put/seqid=0 2024-11-10T03:17:03,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110b16c09c7b3c34bd3aea1334317009d49_f294ea12149d12df042ace2dfb55c011 is 50, key is test_row_0/A:col10/1731208622292/Put/seqid=0 2024-11-10T03:17:03,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742054_1230 (size=13255) 2024-11-10T03:17:04,012 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/B/406abe1f75c440edbc3f2d37c77d944e as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/406abe1f75c440edbc3f2d37c77d944e 2024-11-10T03:17:04,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742055_1231 (size=12454) 2024-11-10T03:17:04,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:04,023 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110b16c09c7b3c34bd3aea1334317009d49_f294ea12149d12df042ace2dfb55c011 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110b16c09c7b3c34bd3aea1334317009d49_f294ea12149d12df042ace2dfb55c011 2024-11-10T03:17:04,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/A/5587ee66899741dbbda26f60f2425a47, store: [table=TestAcidGuarantees family=A region=f294ea12149d12df042ace2dfb55c011] 2024-11-10T03:17:04,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/A/5587ee66899741dbbda26f60f2425a47 is 175, key is test_row_0/A:col10/1731208622292/Put/seqid=0 2024-11-10T03:17:04,030 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f294ea12149d12df042ace2dfb55c011] 2024-11-10T03:17:04,030 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f294ea12149d12df042ace2dfb55c011/B of f294ea12149d12df042ace2dfb55c011 into 406abe1f75c440edbc3f2d37c77d944e(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:17:04,030 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f294ea12149d12df042ace2dfb55c011: 2024-11-10T03:17:04,030 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., storeName=f294ea12149d12df042ace2dfb55c011/B, priority=13, startTime=1731208623946; duration=0sec 2024-11-10T03:17:04,030 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T03:17:04,030 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f294ea12149d12df042ace2dfb55c011:B 2024-11-10T03:17:04,031 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T03:17:04,032 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T03:17:04,032 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1541): f294ea12149d12df042ace2dfb55c011/C is initiating minor compaction (all files) 2024-11-10T03:17:04,032 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f294ea12149d12df042ace2dfb55c011/C in TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:17:04,032 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/d5a8ec1471ca423d952573202a16edca, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/00d998eb554240b1bfaaa07b002c6c8b, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/831f246d504d437e994397c3751e9fd0] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp, totalSize=36.9 K 2024-11-10T03:17:04,033 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241110d65cafbedcfc40efbe220cc292d1418c_f294ea12149d12df042ace2dfb55c011 store=[table=TestAcidGuarantees family=A region=f294ea12149d12df042ace2dfb55c011] 2024-11-10T03:17:04,033 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting d5a8ec1471ca423d952573202a16edca, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=386, earliestPutTs=1731208619270 2024-11-10T03:17:04,035 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 00d998eb554240b1bfaaa07b002c6c8b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1731208619906 2024-11-10T03:17:04,035 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 831f246d504d437e994397c3751e9fd0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=446, earliestPutTs=1731208622201 2024-11-10T03:17:04,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742056_1232 (size=31255) 2024-11-10T03:17:04,053 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=458, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/A/5587ee66899741dbbda26f60f2425a47 2024-11-10T03:17:04,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/B/e6237b6f66f446b195899b4f268c12ce is 50, key is test_row_0/B:col10/1731208622292/Put/seqid=0 2024-11-10T03:17:04,080 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f294ea12149d12df042ace2dfb55c011#C#compaction#190 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:17:04,081 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/C/cfe955c905904d7e95815016d0dc8d5d is 50, key is test_row_0/C:col10/1731208622201/Put/seqid=0 2024-11-10T03:17:04,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742057_1233 (size=12301) 2024-11-10T03:17:04,114 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241110d65cafbedcfc40efbe220cc292d1418c_f294ea12149d12df042ace2dfb55c011, store=[table=TestAcidGuarantees family=A region=f294ea12149d12df042ace2dfb55c011] 2024-11-10T03:17:04,114 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110d65cafbedcfc40efbe220cc292d1418c_f294ea12149d12df042ace2dfb55c011 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f294ea12149d12df042ace2dfb55c011] 2024-11-10T03:17:04,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742058_1234 (size=13255) 2024-11-10T03:17:04,131 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/C/cfe955c905904d7e95815016d0dc8d5d as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/cfe955c905904d7e95815016d0dc8d5d 2024-11-10T03:17:04,139 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f294ea12149d12df042ace2dfb55c011/C of f294ea12149d12df042ace2dfb55c011 into cfe955c905904d7e95815016d0dc8d5d(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:17:04,139 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f294ea12149d12df042ace2dfb55c011: 2024-11-10T03:17:04,139 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., storeName=f294ea12149d12df042ace2dfb55c011/C, priority=13, startTime=1731208623947; duration=0sec 2024-11-10T03:17:04,139 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:17:04,139 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f294ea12149d12df042ace2dfb55c011:C 2024-11-10T03:17:04,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742059_1235 (size=4469) 2024-11-10T03:17:04,147 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f294ea12149d12df042ace2dfb55c011#A#compaction#188 average throughput is 0.21 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:17:04,147 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/A/784a8dd49a894045b542508ffbbf2e8c is 175, key is test_row_0/A:col10/1731208622201/Put/seqid=0 2024-11-10T03:17:04,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742060_1236 (size=32209) 2024-11-10T03:17:04,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-10T03:17:04,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on f294ea12149d12df042ace2dfb55c011 2024-11-10T03:17:04,455 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. as already flushing 2024-11-10T03:17:04,455 INFO [Thread-606 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-606. 2024-11-10T03:17:04,455 DEBUG [Thread-606 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T03:17:04,455 DEBUG [Thread-606 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:17:04,455 DEBUG [Thread-606 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:17:04,455 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T03:17:04,461 INFO [Thread-602 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-602. 2024-11-10T03:17:04,461 DEBUG [Thread-602 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T03:17:04,461 DEBUG [Thread-602 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:17:04,461 DEBUG [Thread-602 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:17:04,461 INFO [Thread-600 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-600. 2024-11-10T03:17:04,461 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T03:17:04,461 DEBUG [Thread-600 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T03:17:04,462 DEBUG [Thread-600 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:17:04,462 DEBUG [Thread-600 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:17:04,462 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T03:17:04,514 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=458 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/B/e6237b6f66f446b195899b4f268c12ce 2024-11-10T03:17:04,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/C/412a73f0a6a54deb9a1340a399d2d8ac is 50, key is test_row_0/C:col10/1731208622292/Put/seqid=0 2024-11-10T03:17:04,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742061_1237 (size=12301) 2024-11-10T03:17:04,588 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/A/784a8dd49a894045b542508ffbbf2e8c as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/784a8dd49a894045b542508ffbbf2e8c 2024-11-10T03:17:04,608 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f294ea12149d12df042ace2dfb55c011/A of f294ea12149d12df042ace2dfb55c011 into 784a8dd49a894045b542508ffbbf2e8c(size=31.5 K), total size for store is 31.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:17:04,608 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f294ea12149d12df042ace2dfb55c011: 2024-11-10T03:17:04,608 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., storeName=f294ea12149d12df042ace2dfb55c011/A, priority=13, startTime=1731208623946; duration=0sec 2024-11-10T03:17:04,608 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:17:04,608 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f294ea12149d12df042ace2dfb55c011:A 2024-11-10T03:17:04,955 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=458 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/C/412a73f0a6a54deb9a1340a399d2d8ac 2024-11-10T03:17:04,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/A/5587ee66899741dbbda26f60f2425a47 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/5587ee66899741dbbda26f60f2425a47 2024-11-10T03:17:04,965 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/5587ee66899741dbbda26f60f2425a47, entries=150, sequenceid=458, filesize=30.5 K 2024-11-10T03:17:04,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/B/e6237b6f66f446b195899b4f268c12ce as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/e6237b6f66f446b195899b4f268c12ce 2024-11-10T03:17:04,975 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/e6237b6f66f446b195899b4f268c12ce, entries=150, sequenceid=458, filesize=12.0 K 2024-11-10T03:17:04,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/C/412a73f0a6a54deb9a1340a399d2d8ac as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/412a73f0a6a54deb9a1340a399d2d8ac 2024-11-10T03:17:04,983 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/412a73f0a6a54deb9a1340a399d2d8ac, entries=150, sequenceid=458, filesize=12.0 K 2024-11-10T03:17:04,984 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=20.13 KB/20610 for f294ea12149d12df042ace2dfb55c011 in 1032ms, sequenceid=458, compaction requested=false 2024-11-10T03:17:04,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for f294ea12149d12df042ace2dfb55c011: 2024-11-10T03:17:04,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:17:04,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-10T03:17:04,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-10T03:17:04,989 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-10T03:17:04,989 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7280 sec 2024-11-10T03:17:04,991 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 2.7350 sec 2024-11-10T03:17:06,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-10T03:17:06,423 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T03:17:09,635 INFO [Thread-604 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-604. 2024-11-10T03:17:09,635 INFO [Thread-598 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-598. 2024-11-10T03:17:09,635 DEBUG [Thread-604 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T03:17:09,635 DEBUG [Thread-598 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T03:17:09,635 DEBUG [Thread-604 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:17:09,635 DEBUG [Thread-598 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:17:09,635 DEBUG [Thread-604 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:17:09,635 DEBUG [Thread-598 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:17:09,635 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T03:17:09,635 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T03:17:09,635 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-10T03:17:09,635 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 39 2024-11-10T03:17:09,636 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 109 2024-11-10T03:17:09,636 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 93 2024-11-10T03:17:09,636 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 37 2024-11-10T03:17:09,636 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 94 2024-11-10T03:17:09,636 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-10T03:17:09,636 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2343 2024-11-10T03:17:09,636 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2304 2024-11-10T03:17:09,636 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-10T03:17:09,636 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1044 2024-11-10T03:17:09,636 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3129 rows 2024-11-10T03:17:09,636 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1053 2024-11-10T03:17:09,636 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3156 rows 2024-11-10T03:17:09,636 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T03:17:09,636 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T03:17:09,636 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:17:09,636 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:17:09,636 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T03:17:09,637 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-10T03:17:09,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-10T03:17:09,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-10T03:17:09,640 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731208629640"}]},"ts":"1731208629640"} 2024-11-10T03:17:09,642 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-10T03:17:09,643 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-10T03:17:09,644 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-10T03:17:09,647 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f294ea12149d12df042ace2dfb55c011, UNASSIGN}] 2024-11-10T03:17:09,649 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f294ea12149d12df042ace2dfb55c011, UNASSIGN 2024-11-10T03:17:09,651 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=f294ea12149d12df042ace2dfb55c011, regionState=CLOSING, regionLocation=394d2222f724,38139,1731208572054 2024-11-10T03:17:09,654 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f294ea12149d12df042ace2dfb55c011, UNASSIGN because future has completed 2024-11-10T03:17:09,655 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-10T03:17:09,655 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; CloseRegionProcedure f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054}] 2024-11-10T03:17:09,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-10T03:17:09,808 INFO [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(122): Close f294ea12149d12df042ace2dfb55c011 2024-11-10T03:17:09,808 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-10T03:17:09,809 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1722): Closing f294ea12149d12df042ace2dfb55c011, disabling compactions & flushes 2024-11-10T03:17:09,809 INFO [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:17:09,809 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:17:09,809 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. after waiting 0 ms 2024-11-10T03:17:09,809 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:17:09,809 INFO [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(2902): Flushing f294ea12149d12df042ace2dfb55c011 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-10T03:17:09,809 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f294ea12149d12df042ace2dfb55c011, store=A 2024-11-10T03:17:09,809 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:17:09,809 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f294ea12149d12df042ace2dfb55c011, store=B 2024-11-10T03:17:09,809 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:17:09,809 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f294ea12149d12df042ace2dfb55c011, store=C 2024-11-10T03:17:09,809 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:17:09,818 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110df3847ef39fe4f138a65033e58a1da08_f294ea12149d12df042ace2dfb55c011 is 50, key is test_row_0/A:col10/1731208629634/Put/seqid=0 2024-11-10T03:17:09,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742062_1238 (size=9914) 2024-11-10T03:17:09,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-10T03:17:10,223 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:10,231 INFO [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110df3847ef39fe4f138a65033e58a1da08_f294ea12149d12df042ace2dfb55c011 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110df3847ef39fe4f138a65033e58a1da08_f294ea12149d12df042ace2dfb55c011 2024-11-10T03:17:10,233 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/A/b393b5d490e64c4eb4f9379cc6f53356, store: [table=TestAcidGuarantees family=A region=f294ea12149d12df042ace2dfb55c011] 2024-11-10T03:17:10,233 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/A/b393b5d490e64c4eb4f9379cc6f53356 is 175, key is test_row_0/A:col10/1731208629634/Put/seqid=0 2024-11-10T03:17:10,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742063_1239 (size=22561) 2024-11-10T03:17:10,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-10T03:17:10,311 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-10T03:17:10,638 INFO [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=469, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/A/b393b5d490e64c4eb4f9379cc6f53356 2024-11-10T03:17:10,648 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/B/63d6a43495eb4ba78efe775e6e432233 is 50, key is test_row_0/B:col10/1731208629634/Put/seqid=0 2024-11-10T03:17:10,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742064_1240 (size=9857) 2024-11-10T03:17:10,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-10T03:17:11,054 INFO [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=469 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/B/63d6a43495eb4ba78efe775e6e432233 2024-11-10T03:17:11,065 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/C/18de17de768140cc9100b772f6b77ab0 is 50, key is test_row_0/C:col10/1731208629634/Put/seqid=0 2024-11-10T03:17:11,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742065_1241 (size=9857) 2024-11-10T03:17:11,470 INFO [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=469 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/C/18de17de768140cc9100b772f6b77ab0 2024-11-10T03:17:11,476 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/A/b393b5d490e64c4eb4f9379cc6f53356 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/b393b5d490e64c4eb4f9379cc6f53356 2024-11-10T03:17:11,480 INFO [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/b393b5d490e64c4eb4f9379cc6f53356, entries=100, sequenceid=469, filesize=22.0 K 2024-11-10T03:17:11,481 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/B/63d6a43495eb4ba78efe775e6e432233 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/63d6a43495eb4ba78efe775e6e432233 2024-11-10T03:17:11,486 INFO [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/63d6a43495eb4ba78efe775e6e432233, entries=100, sequenceid=469, filesize=9.6 K 2024-11-10T03:17:11,487 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/.tmp/C/18de17de768140cc9100b772f6b77ab0 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/18de17de768140cc9100b772f6b77ab0 2024-11-10T03:17:11,492 INFO [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/18de17de768140cc9100b772f6b77ab0, entries=100, sequenceid=469, filesize=9.6 K 2024-11-10T03:17:11,494 INFO [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for f294ea12149d12df042ace2dfb55c011 in 1684ms, sequenceid=469, compaction requested=true 2024-11-10T03:17:11,494 DEBUG [StoreCloser-TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/336f3beea4234c409697878d7c1cf6e0, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/1a7935230a7e4260a394104d9b454d4d, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/8a7f7bb8184848e29d35fc61500a2500, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/8918c6ddff4d44929bc95906305cbbef, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/7b5fa90e6cf34eacaed35db4e981d48f, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/6ecb82eaee404670b0fea6a033ab7d3f, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/d94adcb6627d447399754c787e225d93, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/ff659bfa7d7e445db204649d842022a2, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/c29ea343310b4db3801e38b22c82f03c, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/cad986c7afc645a5b578e01c9bf4ab7b, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/8045e9cb51c64fe9bc686c5f7bca09a8, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/c0d3a7083abb4ece9ccd6935a6f0a607, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/1524fa518d1247a6b238cce56699e62e, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/c2eda5347b6f417baa61a5abd3987da7, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/cf8421c4775b4ce89b65c91472a8f189, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/7e5c546522dc4a8890790c771a6e103e, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/e38a362af6ae4832804fde301289c786, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/d7a444cd57574b61b9d1c72a79eb207d, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/bcc9230d27b14b59b579b86057f85d6f, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/6a9376887ebb42bd82b58762ca9ba345, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/0690eb4c09d04350b1ca995efd66b193, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/1ae3dc7a46cb415e8ae950737750eb55, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/632e920b49b94953bb1a920b261ca240, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/4bc11ee82f3b4306a8e1ef0cc0c5b164, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/311f5bb07518487a94dc40e1df08b0a4, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/a38211a9e93d4e659d90f07e66879b4e, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/dca4c9d025e84bda86f62d4ef16a31cc, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/a00853f4a2a240bf882f1b82dca2fb40] to archive 2024-11-10T03:17:11,496 DEBUG [StoreCloser-TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T03:17:11,498 DEBUG [StoreCloser-TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/336f3beea4234c409697878d7c1cf6e0 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/336f3beea4234c409697878d7c1cf6e0 2024-11-10T03:17:11,499 DEBUG [StoreCloser-TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/1a7935230a7e4260a394104d9b454d4d to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/1a7935230a7e4260a394104d9b454d4d 2024-11-10T03:17:11,500 DEBUG [StoreCloser-TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/8a7f7bb8184848e29d35fc61500a2500 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/8a7f7bb8184848e29d35fc61500a2500 2024-11-10T03:17:11,502 DEBUG [StoreCloser-TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/8918c6ddff4d44929bc95906305cbbef to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/8918c6ddff4d44929bc95906305cbbef 2024-11-10T03:17:11,503 DEBUG [StoreCloser-TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/7b5fa90e6cf34eacaed35db4e981d48f to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/7b5fa90e6cf34eacaed35db4e981d48f 2024-11-10T03:17:11,505 DEBUG [StoreCloser-TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/6ecb82eaee404670b0fea6a033ab7d3f to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/6ecb82eaee404670b0fea6a033ab7d3f 2024-11-10T03:17:11,506 DEBUG [StoreCloser-TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/d94adcb6627d447399754c787e225d93 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/d94adcb6627d447399754c787e225d93 2024-11-10T03:17:11,507 DEBUG [StoreCloser-TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/ff659bfa7d7e445db204649d842022a2 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/ff659bfa7d7e445db204649d842022a2 2024-11-10T03:17:11,509 DEBUG [StoreCloser-TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/c29ea343310b4db3801e38b22c82f03c to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/c29ea343310b4db3801e38b22c82f03c 2024-11-10T03:17:11,510 DEBUG [StoreCloser-TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/cad986c7afc645a5b578e01c9bf4ab7b to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/cad986c7afc645a5b578e01c9bf4ab7b 2024-11-10T03:17:11,511 DEBUG [StoreCloser-TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/8045e9cb51c64fe9bc686c5f7bca09a8 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/8045e9cb51c64fe9bc686c5f7bca09a8 2024-11-10T03:17:11,512 DEBUG [StoreCloser-TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/c0d3a7083abb4ece9ccd6935a6f0a607 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/c0d3a7083abb4ece9ccd6935a6f0a607 2024-11-10T03:17:11,513 DEBUG [StoreCloser-TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/1524fa518d1247a6b238cce56699e62e to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/1524fa518d1247a6b238cce56699e62e 2024-11-10T03:17:11,514 DEBUG [StoreCloser-TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/c2eda5347b6f417baa61a5abd3987da7 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/c2eda5347b6f417baa61a5abd3987da7 2024-11-10T03:17:11,515 DEBUG [StoreCloser-TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/cf8421c4775b4ce89b65c91472a8f189 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/cf8421c4775b4ce89b65c91472a8f189 2024-11-10T03:17:11,516 DEBUG [StoreCloser-TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/7e5c546522dc4a8890790c771a6e103e to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/7e5c546522dc4a8890790c771a6e103e 2024-11-10T03:17:11,517 DEBUG [StoreCloser-TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/e38a362af6ae4832804fde301289c786 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/e38a362af6ae4832804fde301289c786 2024-11-10T03:17:11,518 DEBUG [StoreCloser-TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/d7a444cd57574b61b9d1c72a79eb207d to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/d7a444cd57574b61b9d1c72a79eb207d 2024-11-10T03:17:11,519 DEBUG [StoreCloser-TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/bcc9230d27b14b59b579b86057f85d6f to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/bcc9230d27b14b59b579b86057f85d6f 2024-11-10T03:17:11,519 DEBUG [StoreCloser-TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/6a9376887ebb42bd82b58762ca9ba345 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/6a9376887ebb42bd82b58762ca9ba345 2024-11-10T03:17:11,520 DEBUG [StoreCloser-TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/0690eb4c09d04350b1ca995efd66b193 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/0690eb4c09d04350b1ca995efd66b193 2024-11-10T03:17:11,521 DEBUG [StoreCloser-TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/1ae3dc7a46cb415e8ae950737750eb55 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/1ae3dc7a46cb415e8ae950737750eb55 2024-11-10T03:17:11,522 DEBUG [StoreCloser-TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/632e920b49b94953bb1a920b261ca240 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/632e920b49b94953bb1a920b261ca240 2024-11-10T03:17:11,523 DEBUG [StoreCloser-TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/4bc11ee82f3b4306a8e1ef0cc0c5b164 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/4bc11ee82f3b4306a8e1ef0cc0c5b164 2024-11-10T03:17:11,524 DEBUG [StoreCloser-TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/311f5bb07518487a94dc40e1df08b0a4 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/311f5bb07518487a94dc40e1df08b0a4 2024-11-10T03:17:11,525 DEBUG [StoreCloser-TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/a38211a9e93d4e659d90f07e66879b4e to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/a38211a9e93d4e659d90f07e66879b4e 2024-11-10T03:17:11,526 DEBUG [StoreCloser-TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/dca4c9d025e84bda86f62d4ef16a31cc to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/dca4c9d025e84bda86f62d4ef16a31cc 2024-11-10T03:17:11,527 DEBUG [StoreCloser-TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/a00853f4a2a240bf882f1b82dca2fb40 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/a00853f4a2a240bf882f1b82dca2fb40 2024-11-10T03:17:11,529 DEBUG [StoreCloser-TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/213d7f1b07a94046a71959658c84ed06, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/0fdf4bd8187246029f31ea7c9beb3d07, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/644149eabe39456dadfa1936ebc2f837, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/421e2f2d43d04adabd659427d7489107, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/2066322ace80471586cd0f8112f93c4d, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/71a175df075c424fa4d52439da8eb5fc, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/a92fbcaade63481392676c6bbb53a0d6, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/be3e0aa1bcfd4b1db9280d355a709946, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/7d5e517c23014f818f98108d8254a81d, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/31fb514dd41a43c58ce6eec10b0d08d0, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/14bbcbf2779b4e48b38ac43f5e02e428, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/0aa5edd7d5024eb9866743af026efd58, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/96a1b69b41b346d3a51de986f587f83a, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/d21fc88b51cf42c3802cf48b86fa9a78, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/60e612dae1374441a48c7efb3ad6103b, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/bb654c7282eb4cd7bdc37bf3c831d7fc, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/a3db52a6578b41f7886373720b94b32b, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/d2e10726678f411d91cfd9d550a14209, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/5cd6128793ef415a944938f8659be45f, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/12332906f4fa4559ba1d1dcb7d756d47, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/728939e45b6447189c598546d97266ba, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/22cbab8b0d5d42f1985065431b1780f8, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/7db358bd4959452b8c8cc6ce96409b32, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/863e68d63eca4b52928535415f3b5da2, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/54f360ed8c6b487e866b89aa402a0978, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/9e19e1cf342a461ba6ca3529bd03b6db, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/3c053d4d3fff468aaa4dfea08976ee51, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/38b876ae322c44aea62d839bfcc42eb6] to archive 2024-11-10T03:17:11,530 DEBUG [StoreCloser-TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T03:17:11,532 DEBUG [StoreCloser-TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/213d7f1b07a94046a71959658c84ed06 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/213d7f1b07a94046a71959658c84ed06 2024-11-10T03:17:11,533 DEBUG [StoreCloser-TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/0fdf4bd8187246029f31ea7c9beb3d07 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/0fdf4bd8187246029f31ea7c9beb3d07 2024-11-10T03:17:11,534 DEBUG [StoreCloser-TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/644149eabe39456dadfa1936ebc2f837 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/644149eabe39456dadfa1936ebc2f837 2024-11-10T03:17:11,535 DEBUG [StoreCloser-TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/421e2f2d43d04adabd659427d7489107 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/421e2f2d43d04adabd659427d7489107 2024-11-10T03:17:11,536 DEBUG [StoreCloser-TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/2066322ace80471586cd0f8112f93c4d to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/2066322ace80471586cd0f8112f93c4d 2024-11-10T03:17:11,537 DEBUG [StoreCloser-TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/71a175df075c424fa4d52439da8eb5fc to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/71a175df075c424fa4d52439da8eb5fc 2024-11-10T03:17:11,538 DEBUG [StoreCloser-TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/a92fbcaade63481392676c6bbb53a0d6 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/a92fbcaade63481392676c6bbb53a0d6 2024-11-10T03:17:11,539 DEBUG [StoreCloser-TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/be3e0aa1bcfd4b1db9280d355a709946 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/be3e0aa1bcfd4b1db9280d355a709946 2024-11-10T03:17:11,540 DEBUG [StoreCloser-TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/7d5e517c23014f818f98108d8254a81d to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/7d5e517c23014f818f98108d8254a81d 2024-11-10T03:17:11,541 DEBUG [StoreCloser-TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/31fb514dd41a43c58ce6eec10b0d08d0 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/31fb514dd41a43c58ce6eec10b0d08d0 2024-11-10T03:17:11,542 DEBUG [StoreCloser-TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/14bbcbf2779b4e48b38ac43f5e02e428 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/14bbcbf2779b4e48b38ac43f5e02e428 2024-11-10T03:17:11,543 DEBUG [StoreCloser-TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/0aa5edd7d5024eb9866743af026efd58 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/0aa5edd7d5024eb9866743af026efd58 2024-11-10T03:17:11,544 DEBUG [StoreCloser-TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/96a1b69b41b346d3a51de986f587f83a to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/96a1b69b41b346d3a51de986f587f83a 2024-11-10T03:17:11,545 DEBUG [StoreCloser-TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/d21fc88b51cf42c3802cf48b86fa9a78 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/d21fc88b51cf42c3802cf48b86fa9a78 2024-11-10T03:17:11,546 DEBUG [StoreCloser-TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/60e612dae1374441a48c7efb3ad6103b to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/60e612dae1374441a48c7efb3ad6103b 2024-11-10T03:17:11,547 DEBUG [StoreCloser-TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/bb654c7282eb4cd7bdc37bf3c831d7fc to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/bb654c7282eb4cd7bdc37bf3c831d7fc 2024-11-10T03:17:11,548 DEBUG [StoreCloser-TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/a3db52a6578b41f7886373720b94b32b to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/a3db52a6578b41f7886373720b94b32b 2024-11-10T03:17:11,549 DEBUG [StoreCloser-TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/d2e10726678f411d91cfd9d550a14209 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/d2e10726678f411d91cfd9d550a14209 2024-11-10T03:17:11,549 DEBUG [StoreCloser-TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/5cd6128793ef415a944938f8659be45f to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/5cd6128793ef415a944938f8659be45f 2024-11-10T03:17:11,550 DEBUG [StoreCloser-TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/12332906f4fa4559ba1d1dcb7d756d47 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/12332906f4fa4559ba1d1dcb7d756d47 2024-11-10T03:17:11,551 DEBUG [StoreCloser-TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/728939e45b6447189c598546d97266ba to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/728939e45b6447189c598546d97266ba 2024-11-10T03:17:11,552 DEBUG [StoreCloser-TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/22cbab8b0d5d42f1985065431b1780f8 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/22cbab8b0d5d42f1985065431b1780f8 2024-11-10T03:17:11,553 DEBUG [StoreCloser-TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/7db358bd4959452b8c8cc6ce96409b32 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/7db358bd4959452b8c8cc6ce96409b32 2024-11-10T03:17:11,554 DEBUG [StoreCloser-TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/863e68d63eca4b52928535415f3b5da2 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/863e68d63eca4b52928535415f3b5da2 2024-11-10T03:17:11,555 DEBUG [StoreCloser-TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/54f360ed8c6b487e866b89aa402a0978 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/54f360ed8c6b487e866b89aa402a0978 2024-11-10T03:17:11,555 DEBUG [StoreCloser-TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/9e19e1cf342a461ba6ca3529bd03b6db to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/9e19e1cf342a461ba6ca3529bd03b6db 2024-11-10T03:17:11,556 DEBUG [StoreCloser-TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/3c053d4d3fff468aaa4dfea08976ee51 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/3c053d4d3fff468aaa4dfea08976ee51 2024-11-10T03:17:11,557 DEBUG [StoreCloser-TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/38b876ae322c44aea62d839bfcc42eb6 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/38b876ae322c44aea62d839bfcc42eb6 2024-11-10T03:17:11,559 DEBUG [StoreCloser-TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/3d2fc9f3c23846299a1f7aae62b47ae1, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/539a5109cd28463a8b5a86145d6aacfd, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/ab2d92ee0a944503abd0ae5c2b615653, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/e457e4e05b6848cca6308952496be592, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/54bb705b4252426a8ec85fbccbe36d02, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/1084770c5a5e4b54ba8de4f63e5b7ada, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/3aca2c39adad4adea221f8c80be0c51e, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/da0d3cb699e3418abcb877159c8fc1d4, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/93e15adcf384445b99e700001c6d0776, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/0deab162ce8d4b3ebbcf348ad6dac0a6, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/d4c320f2ae08487ca6daa1a81f230a02, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/5383bb8043d8467fa677ef5c252776c7, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/67bb0579a8984d0b9b06a2132a9cba1b, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/e2e01ba5043445babdf65324e15dd6e8, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/17a01aa79d214dac9505e287aabec4a0, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/128133b7282b44ff9ce929eb8769d7f0, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/8ee9eaf43087453c89d2dec15bc50129, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/8a690cb9d0f84df4ac4d2c37e75893ef, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/137e6978efea4f138aeac3c3d511f789, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/5bc9d89143594d0bb42e955ec7d8114c, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/07b7a6e240b24047ac0b5638ba5e4894, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/abd12603d8a944bbac02793dd577f871, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/90738642720c422f8bca2f33db574a2a, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/1457d735e0cf45fabc9f4b611448690e, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/d5a8ec1471ca423d952573202a16edca, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/43e81a29b3a14879ae1e8ef35a2964d1, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/00d998eb554240b1bfaaa07b002c6c8b, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/831f246d504d437e994397c3751e9fd0] to archive 2024-11-10T03:17:11,559 DEBUG [StoreCloser-TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T03:17:11,561 DEBUG [StoreCloser-TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/3d2fc9f3c23846299a1f7aae62b47ae1 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/3d2fc9f3c23846299a1f7aae62b47ae1 2024-11-10T03:17:11,562 DEBUG [StoreCloser-TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/539a5109cd28463a8b5a86145d6aacfd to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/539a5109cd28463a8b5a86145d6aacfd 2024-11-10T03:17:11,564 DEBUG [StoreCloser-TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/ab2d92ee0a944503abd0ae5c2b615653 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/ab2d92ee0a944503abd0ae5c2b615653 2024-11-10T03:17:11,565 DEBUG [StoreCloser-TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/e457e4e05b6848cca6308952496be592 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/e457e4e05b6848cca6308952496be592 2024-11-10T03:17:11,566 DEBUG [StoreCloser-TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/54bb705b4252426a8ec85fbccbe36d02 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/54bb705b4252426a8ec85fbccbe36d02 2024-11-10T03:17:11,567 DEBUG [StoreCloser-TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/1084770c5a5e4b54ba8de4f63e5b7ada to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/1084770c5a5e4b54ba8de4f63e5b7ada 2024-11-10T03:17:11,568 DEBUG [StoreCloser-TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/3aca2c39adad4adea221f8c80be0c51e to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/3aca2c39adad4adea221f8c80be0c51e 2024-11-10T03:17:11,570 DEBUG [StoreCloser-TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/da0d3cb699e3418abcb877159c8fc1d4 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/da0d3cb699e3418abcb877159c8fc1d4 2024-11-10T03:17:11,571 DEBUG [StoreCloser-TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/93e15adcf384445b99e700001c6d0776 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/93e15adcf384445b99e700001c6d0776 2024-11-10T03:17:11,572 DEBUG [StoreCloser-TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/0deab162ce8d4b3ebbcf348ad6dac0a6 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/0deab162ce8d4b3ebbcf348ad6dac0a6 2024-11-10T03:17:11,573 DEBUG [StoreCloser-TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/d4c320f2ae08487ca6daa1a81f230a02 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/d4c320f2ae08487ca6daa1a81f230a02 2024-11-10T03:17:11,575 DEBUG [StoreCloser-TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/5383bb8043d8467fa677ef5c252776c7 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/5383bb8043d8467fa677ef5c252776c7 2024-11-10T03:17:11,577 DEBUG [StoreCloser-TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/67bb0579a8984d0b9b06a2132a9cba1b to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/67bb0579a8984d0b9b06a2132a9cba1b 2024-11-10T03:17:11,579 DEBUG [StoreCloser-TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/e2e01ba5043445babdf65324e15dd6e8 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/e2e01ba5043445babdf65324e15dd6e8 2024-11-10T03:17:11,580 DEBUG [StoreCloser-TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/17a01aa79d214dac9505e287aabec4a0 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/17a01aa79d214dac9505e287aabec4a0 2024-11-10T03:17:11,581 DEBUG [StoreCloser-TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/128133b7282b44ff9ce929eb8769d7f0 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/128133b7282b44ff9ce929eb8769d7f0 2024-11-10T03:17:11,582 DEBUG [StoreCloser-TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/8ee9eaf43087453c89d2dec15bc50129 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/8ee9eaf43087453c89d2dec15bc50129 2024-11-10T03:17:11,583 DEBUG [StoreCloser-TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/8a690cb9d0f84df4ac4d2c37e75893ef to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/8a690cb9d0f84df4ac4d2c37e75893ef 2024-11-10T03:17:11,584 DEBUG [StoreCloser-TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/137e6978efea4f138aeac3c3d511f789 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/137e6978efea4f138aeac3c3d511f789 2024-11-10T03:17:11,585 DEBUG [StoreCloser-TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/5bc9d89143594d0bb42e955ec7d8114c to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/5bc9d89143594d0bb42e955ec7d8114c 2024-11-10T03:17:11,586 DEBUG [StoreCloser-TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/07b7a6e240b24047ac0b5638ba5e4894 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/07b7a6e240b24047ac0b5638ba5e4894 2024-11-10T03:17:11,587 DEBUG [StoreCloser-TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/abd12603d8a944bbac02793dd577f871 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/abd12603d8a944bbac02793dd577f871 2024-11-10T03:17:11,588 DEBUG [StoreCloser-TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/90738642720c422f8bca2f33db574a2a to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/90738642720c422f8bca2f33db574a2a 2024-11-10T03:17:11,589 DEBUG [StoreCloser-TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/1457d735e0cf45fabc9f4b611448690e to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/1457d735e0cf45fabc9f4b611448690e 2024-11-10T03:17:11,590 DEBUG [StoreCloser-TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/d5a8ec1471ca423d952573202a16edca to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/d5a8ec1471ca423d952573202a16edca 2024-11-10T03:17:11,591 DEBUG [StoreCloser-TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/43e81a29b3a14879ae1e8ef35a2964d1 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/43e81a29b3a14879ae1e8ef35a2964d1 2024-11-10T03:17:11,592 DEBUG [StoreCloser-TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/00d998eb554240b1bfaaa07b002c6c8b to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/00d998eb554240b1bfaaa07b002c6c8b 2024-11-10T03:17:11,593 DEBUG [StoreCloser-TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/831f246d504d437e994397c3751e9fd0 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/831f246d504d437e994397c3751e9fd0 2024-11-10T03:17:11,597 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/recovered.edits/472.seqid, newMaxSeqId=472, maxSeqId=4 2024-11-10T03:17:11,598 INFO [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. 2024-11-10T03:17:11,598 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1676): Region close journal for f294ea12149d12df042ace2dfb55c011: Waiting for close lock at 1731208629808Running coprocessor pre-close hooks at 1731208629808Disabling compacts and flushes for region at 1731208629808Disabling writes for close at 1731208629809 (+1 ms)Obtaining lock to block concurrent updates at 1731208629809Preparing flush snapshotting stores in f294ea12149d12df042ace2dfb55c011 at 1731208629809Finished memstore snapshotting TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731208629809Flushing stores of TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011. at 1731208629810 (+1 ms)Flushing f294ea12149d12df042ace2dfb55c011/A: creating writer at 1731208629810Flushing mob file f294ea12149d12df042ace2dfb55c011/A: appending metadata at 1731208629818 (+8 ms)Flushing mob file f294ea12149d12df042ace2dfb55c011/A: closing flushed file at 1731208629818Flushing f294ea12149d12df042ace2dfb55c011/A: appending metadata at 1731208630233 (+415 ms)Flushing f294ea12149d12df042ace2dfb55c011/A: closing flushed file at 1731208630233Flushing f294ea12149d12df042ace2dfb55c011/B: creating writer at 1731208630643 (+410 ms)Flushing f294ea12149d12df042ace2dfb55c011/B: appending metadata at 1731208630648 (+5 ms)Flushing f294ea12149d12df042ace2dfb55c011/B: closing flushed file at 1731208630648Flushing f294ea12149d12df042ace2dfb55c011/C: creating writer at 1731208631061 (+413 ms)Flushing f294ea12149d12df042ace2dfb55c011/C: appending metadata at 1731208631065 (+4 ms)Flushing f294ea12149d12df042ace2dfb55c011/C: closing flushed file at 1731208631065Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7f6f9fdf: reopening flushed file at 1731208631475 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6ec07155: reopening flushed file at 1731208631480 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@135376d3: reopening flushed file at 1731208631486 (+6 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for f294ea12149d12df042ace2dfb55c011 in 1684ms, sequenceid=469, compaction requested=true at 1731208631494 (+8 ms)Writing region close event to WAL at 1731208631594 (+100 ms)Running coprocessor post-close hooks at 1731208631597 (+3 ms)Closed at 1731208631598 (+1 ms) 2024-11-10T03:17:11,600 INFO [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(157): Closed f294ea12149d12df042ace2dfb55c011 2024-11-10T03:17:11,600 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=f294ea12149d12df042ace2dfb55c011, regionState=CLOSED 2024-11-10T03:17:11,601 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=57, ppid=56, state=RUNNABLE, hasLock=false; CloseRegionProcedure f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 because future has completed 2024-11-10T03:17:11,603 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-10T03:17:11,603 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; CloseRegionProcedure f294ea12149d12df042ace2dfb55c011, server=394d2222f724,38139,1731208572054 in 1.9470 sec 2024-11-10T03:17:11,605 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=55 2024-11-10T03:17:11,605 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=55, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f294ea12149d12df042ace2dfb55c011, UNASSIGN in 1.9560 sec 2024-11-10T03:17:11,607 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-10T03:17:11,607 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9610 sec 2024-11-10T03:17:11,608 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731208631608"}]},"ts":"1731208631608"} 2024-11-10T03:17:11,609 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-10T03:17:11,609 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-10T03:17:11,611 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9730 sec 2024-11-10T03:17:11,664 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-10T03:17:11,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-10T03:17:11,783 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-10T03:17:11,784 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-10T03:17:11,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T03:17:11,785 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=58, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T03:17:11,786 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=58, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T03:17:11,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-10T03:17:11,788 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011 2024-11-10T03:17:11,791 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A, FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B, FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C, FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/recovered.edits] 2024-11-10T03:17:11,794 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/5587ee66899741dbbda26f60f2425a47 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/5587ee66899741dbbda26f60f2425a47 2024-11-10T03:17:11,795 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/784a8dd49a894045b542508ffbbf2e8c to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/784a8dd49a894045b542508ffbbf2e8c 2024-11-10T03:17:11,796 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/b393b5d490e64c4eb4f9379cc6f53356 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/A/b393b5d490e64c4eb4f9379cc6f53356 2024-11-10T03:17:11,798 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/406abe1f75c440edbc3f2d37c77d944e to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/406abe1f75c440edbc3f2d37c77d944e 2024-11-10T03:17:11,799 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/63d6a43495eb4ba78efe775e6e432233 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/63d6a43495eb4ba78efe775e6e432233 2024-11-10T03:17:11,800 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/e6237b6f66f446b195899b4f268c12ce to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/B/e6237b6f66f446b195899b4f268c12ce 2024-11-10T03:17:11,802 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/18de17de768140cc9100b772f6b77ab0 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/18de17de768140cc9100b772f6b77ab0 2024-11-10T03:17:11,803 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/412a73f0a6a54deb9a1340a399d2d8ac to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/412a73f0a6a54deb9a1340a399d2d8ac 2024-11-10T03:17:11,804 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/cfe955c905904d7e95815016d0dc8d5d to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/C/cfe955c905904d7e95815016d0dc8d5d 2024-11-10T03:17:11,807 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/recovered.edits/472.seqid to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011/recovered.edits/472.seqid 2024-11-10T03:17:11,808 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/f294ea12149d12df042ace2dfb55c011 2024-11-10T03:17:11,808 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-10T03:17:11,808 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-10T03:17:11,809 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-10T03:17:11,813 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111020b65f3c93844b0586468f9227a93f79_f294ea12149d12df042ace2dfb55c011 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111020b65f3c93844b0586468f9227a93f79_f294ea12149d12df042ace2dfb55c011 2024-11-10T03:17:11,814 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111032b14e7cd820457a9277167dabcd1407_f294ea12149d12df042ace2dfb55c011 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111032b14e7cd820457a9277167dabcd1407_f294ea12149d12df042ace2dfb55c011 2024-11-10T03:17:11,815 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111034f61ad1ea274020afcb6eb84d2f9028_f294ea12149d12df042ace2dfb55c011 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111034f61ad1ea274020afcb6eb84d2f9028_f294ea12149d12df042ace2dfb55c011 2024-11-10T03:17:11,816 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110408948ca3cfc4b54bfdcff4914f8901c_f294ea12149d12df042ace2dfb55c011 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110408948ca3cfc4b54bfdcff4914f8901c_f294ea12149d12df042ace2dfb55c011 2024-11-10T03:17:11,818 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110546774c3de724b8083d2966ef29818fc_f294ea12149d12df042ace2dfb55c011 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110546774c3de724b8083d2966ef29818fc_f294ea12149d12df042ace2dfb55c011 2024-11-10T03:17:11,819 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110605ab5b011074c139c759ec5db7d5fc0_f294ea12149d12df042ace2dfb55c011 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110605ab5b011074c139c759ec5db7d5fc0_f294ea12149d12df042ace2dfb55c011 2024-11-10T03:17:11,820 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411106d99db466f3d47489c0c1c4a43f2405e_f294ea12149d12df042ace2dfb55c011 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411106d99db466f3d47489c0c1c4a43f2405e_f294ea12149d12df042ace2dfb55c011 2024-11-10T03:17:11,821 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411106e9af77f38934557b860b6fc2dd67835_f294ea12149d12df042ace2dfb55c011 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411106e9af77f38934557b860b6fc2dd67835_f294ea12149d12df042ace2dfb55c011 2024-11-10T03:17:11,823 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111074cbef41c7434b308bab603bfcd3212a_f294ea12149d12df042ace2dfb55c011 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111074cbef41c7434b308bab603bfcd3212a_f294ea12149d12df042ace2dfb55c011 2024-11-10T03:17:11,824 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110861a262bf2304e4eb2d06cadf3f73dee_f294ea12149d12df042ace2dfb55c011 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110861a262bf2304e4eb2d06cadf3f73dee_f294ea12149d12df042ace2dfb55c011 2024-11-10T03:17:11,825 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111099cd0c1f927d471988d0ffeeb5844ff3_f294ea12149d12df042ace2dfb55c011 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111099cd0c1f927d471988d0ffeeb5844ff3_f294ea12149d12df042ace2dfb55c011 2024-11-10T03:17:11,826 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411109f8b2372209b42c683f8216f2c5b121b_f294ea12149d12df042ace2dfb55c011 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411109f8b2372209b42c683f8216f2c5b121b_f294ea12149d12df042ace2dfb55c011 2024-11-10T03:17:11,827 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110a2224e8226e6492e9d3f5bd88434e7cc_f294ea12149d12df042ace2dfb55c011 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110a2224e8226e6492e9d3f5bd88434e7cc_f294ea12149d12df042ace2dfb55c011 2024-11-10T03:17:11,828 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110a95f48bdd687472ebfa049edb22a5413_f294ea12149d12df042ace2dfb55c011 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110a95f48bdd687472ebfa049edb22a5413_f294ea12149d12df042ace2dfb55c011 2024-11-10T03:17:11,829 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110adb34b7b49354c09b5b8a00ee4285801_f294ea12149d12df042ace2dfb55c011 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110adb34b7b49354c09b5b8a00ee4285801_f294ea12149d12df042ace2dfb55c011 2024-11-10T03:17:11,831 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110b09b3ced056a4018bb2bdc812a879e60_f294ea12149d12df042ace2dfb55c011 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110b09b3ced056a4018bb2bdc812a879e60_f294ea12149d12df042ace2dfb55c011 2024-11-10T03:17:11,832 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110b16c09c7b3c34bd3aea1334317009d49_f294ea12149d12df042ace2dfb55c011 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110b16c09c7b3c34bd3aea1334317009d49_f294ea12149d12df042ace2dfb55c011 2024-11-10T03:17:11,833 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110c2d0c77192d846aeb94d3a3fe5861782_f294ea12149d12df042ace2dfb55c011 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110c2d0c77192d846aeb94d3a3fe5861782_f294ea12149d12df042ace2dfb55c011 2024-11-10T03:17:11,834 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110dc3c9ebae322464189b05560544cf756_f294ea12149d12df042ace2dfb55c011 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110dc3c9ebae322464189b05560544cf756_f294ea12149d12df042ace2dfb55c011 2024-11-10T03:17:11,835 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110df3847ef39fe4f138a65033e58a1da08_f294ea12149d12df042ace2dfb55c011 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110df3847ef39fe4f138a65033e58a1da08_f294ea12149d12df042ace2dfb55c011 2024-11-10T03:17:11,836 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110e28a92fe8395480f8df86eac5dd9713f_f294ea12149d12df042ace2dfb55c011 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110e28a92fe8395480f8df86eac5dd9713f_f294ea12149d12df042ace2dfb55c011 2024-11-10T03:17:11,837 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110efa89886bf3a4e8fa899569de7d9ec62_f294ea12149d12df042ace2dfb55c011 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110efa89886bf3a4e8fa899569de7d9ec62_f294ea12149d12df042ace2dfb55c011 2024-11-10T03:17:11,838 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110f0639e8300444c65aea6e1d6adeb5c7e_f294ea12149d12df042ace2dfb55c011 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110f0639e8300444c65aea6e1d6adeb5c7e_f294ea12149d12df042ace2dfb55c011 2024-11-10T03:17:11,838 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-10T03:17:11,840 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=58, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T03:17:11,843 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-10T03:17:11,845 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-10T03:17:11,846 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=58, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T03:17:11,846 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-10T03:17:11,847 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731208631846"}]},"ts":"9223372036854775807"} 2024-11-10T03:17:11,849 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-10T03:17:11,849 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => f294ea12149d12df042ace2dfb55c011, NAME => 'TestAcidGuarantees,,1731208601246.f294ea12149d12df042ace2dfb55c011.', STARTKEY => '', ENDKEY => ''}] 2024-11-10T03:17:11,849 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-10T03:17:11,849 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731208631849"}]},"ts":"9223372036854775807"} 2024-11-10T03:17:11,850 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-10T03:17:11,851 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=58, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T03:17:11,852 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 67 msec 2024-11-10T03:17:11,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-10T03:17:11,893 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-10T03:17:11,893 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-10T03:17:11,906 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=248 (was 250), OpenFileDescriptor=455 (was 457), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=754 (was 769), ProcessCount=11 (was 11), AvailableMemoryMB=1647 (was 2750) 2024-11-10T03:17:11,917 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=248, OpenFileDescriptor=455, MaxFileDescriptor=1048576, SystemLoadAverage=754, ProcessCount=11, AvailableMemoryMB=1647 2024-11-10T03:17:11,918 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-10T03:17:11,918 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T03:17:11,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] procedure2.ProcedureExecutor(1139): Stored pid=59, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-10T03:17:11,920 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-10T03:17:11,920 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:11,921 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 59 2024-11-10T03:17:11,921 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-10T03:17:11,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-10T03:17:11,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742066_1242 (size=1061) 2024-11-10T03:17:12,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-10T03:17:12,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-10T03:17:12,330 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8 2024-11-10T03:17:12,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742067_1243 (size=53) 2024-11-10T03:17:12,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-10T03:17:12,736 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T03:17:12,736 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing a93817da82260912633bd4e3d359d5b7, disabling compactions & flushes 2024-11-10T03:17:12,736 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:12,736 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:12,736 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. after waiting 0 ms 2024-11-10T03:17:12,736 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:12,736 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:12,736 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for a93817da82260912633bd4e3d359d5b7: Waiting for close lock at 1731208632736Disabling compacts and flushes for region at 1731208632736Disabling writes for close at 1731208632736Writing region close event to WAL at 1731208632736Closed at 1731208632736 2024-11-10T03:17:12,737 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-10T03:17:12,737 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731208632737"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731208632737"}]},"ts":"1731208632737"} 2024-11-10T03:17:12,739 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-10T03:17:12,740 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-10T03:17:12,740 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731208632740"}]},"ts":"1731208632740"} 2024-11-10T03:17:12,742 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-10T03:17:12,742 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a93817da82260912633bd4e3d359d5b7, ASSIGN}] 2024-11-10T03:17:12,744 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a93817da82260912633bd4e3d359d5b7, ASSIGN 2024-11-10T03:17:12,745 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a93817da82260912633bd4e3d359d5b7, ASSIGN; state=OFFLINE, location=394d2222f724,38139,1731208572054; forceNewPlan=false, retain=false 2024-11-10T03:17:12,896 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=a93817da82260912633bd4e3d359d5b7, regionState=OPENING, regionLocation=394d2222f724,38139,1731208572054 2024-11-10T03:17:12,901 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a93817da82260912633bd4e3d359d5b7, ASSIGN because future has completed 2024-11-10T03:17:12,901 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; OpenRegionProcedure a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054}] 2024-11-10T03:17:13,061 INFO [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:13,062 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7752): Opening region: {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} 2024-11-10T03:17:13,062 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees a93817da82260912633bd4e3d359d5b7 2024-11-10T03:17:13,062 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T03:17:13,062 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7794): checking encryption for a93817da82260912633bd4e3d359d5b7 2024-11-10T03:17:13,062 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7797): checking classloading for a93817da82260912633bd4e3d359d5b7 2024-11-10T03:17:13,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-10T03:17:13,064 INFO [StoreOpener-a93817da82260912633bd4e3d359d5b7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region a93817da82260912633bd4e3d359d5b7 2024-11-10T03:17:13,065 INFO [StoreOpener-a93817da82260912633bd4e3d359d5b7-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T03:17:13,066 INFO [StoreOpener-a93817da82260912633bd4e3d359d5b7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a93817da82260912633bd4e3d359d5b7 columnFamilyName A 2024-11-10T03:17:13,066 DEBUG [StoreOpener-a93817da82260912633bd4e3d359d5b7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:13,066 INFO [StoreOpener-a93817da82260912633bd4e3d359d5b7-1 {}] regionserver.HStore(327): Store=a93817da82260912633bd4e3d359d5b7/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T03:17:13,066 INFO [StoreOpener-a93817da82260912633bd4e3d359d5b7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region a93817da82260912633bd4e3d359d5b7 2024-11-10T03:17:13,067 INFO [StoreOpener-a93817da82260912633bd4e3d359d5b7-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T03:17:13,068 INFO [StoreOpener-a93817da82260912633bd4e3d359d5b7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a93817da82260912633bd4e3d359d5b7 columnFamilyName B 2024-11-10T03:17:13,068 DEBUG [StoreOpener-a93817da82260912633bd4e3d359d5b7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:13,068 INFO [StoreOpener-a93817da82260912633bd4e3d359d5b7-1 {}] regionserver.HStore(327): Store=a93817da82260912633bd4e3d359d5b7/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T03:17:13,068 INFO [StoreOpener-a93817da82260912633bd4e3d359d5b7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region a93817da82260912633bd4e3d359d5b7 2024-11-10T03:17:13,069 INFO [StoreOpener-a93817da82260912633bd4e3d359d5b7-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T03:17:13,069 INFO [StoreOpener-a93817da82260912633bd4e3d359d5b7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a93817da82260912633bd4e3d359d5b7 columnFamilyName C 2024-11-10T03:17:13,069 DEBUG [StoreOpener-a93817da82260912633bd4e3d359d5b7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:13,070 INFO [StoreOpener-a93817da82260912633bd4e3d359d5b7-1 {}] regionserver.HStore(327): Store=a93817da82260912633bd4e3d359d5b7/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T03:17:13,070 INFO [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:13,070 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1038): replaying wal for a93817da82260912633bd4e3d359d5b7 2024-11-10T03:17:13,070 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7 2024-11-10T03:17:13,071 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7 2024-11-10T03:17:13,071 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1048): stopping wal replay for a93817da82260912633bd4e3d359d5b7 2024-11-10T03:17:13,071 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1060): Cleaning up temporary data for a93817da82260912633bd4e3d359d5b7 2024-11-10T03:17:13,072 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T03:17:13,073 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1093): writing seq id for a93817da82260912633bd4e3d359d5b7 2024-11-10T03:17:13,075 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T03:17:13,075 INFO [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1114): Opened a93817da82260912633bd4e3d359d5b7; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73495489, jitterRate=0.09516812860965729}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T03:17:13,075 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a93817da82260912633bd4e3d359d5b7 2024-11-10T03:17:13,076 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1006): Region open journal for a93817da82260912633bd4e3d359d5b7: Running coprocessor pre-open hook at 1731208633062Writing region info on filesystem at 1731208633062Initializing all the Stores at 1731208633064 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731208633064Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731208633064Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731208633064Cleaning up temporary data from old regions at 1731208633071 (+7 ms)Running coprocessor post-open hooks at 1731208633075 (+4 ms)Region opened successfully at 1731208633076 (+1 ms) 2024-11-10T03:17:13,076 INFO [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., pid=61, masterSystemTime=1731208633054 2024-11-10T03:17:13,078 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:13,078 INFO [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:13,078 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=a93817da82260912633bd4e3d359d5b7, regionState=OPEN, openSeqNum=2, regionLocation=394d2222f724,38139,1731208572054 2024-11-10T03:17:13,080 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=61, ppid=60, state=RUNNABLE, hasLock=false; OpenRegionProcedure a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 because future has completed 2024-11-10T03:17:13,082 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-10T03:17:13,082 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 in 179 msec 2024-11-10T03:17:13,084 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=59 2024-11-10T03:17:13,084 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=59, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a93817da82260912633bd4e3d359d5b7, ASSIGN in 340 msec 2024-11-10T03:17:13,084 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-10T03:17:13,085 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731208633084"}]},"ts":"1731208633084"} 2024-11-10T03:17:13,086 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-10T03:17:13,086 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-10T03:17:13,087 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1680 sec 2024-11-10T03:17:14,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-10T03:17:14,074 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-10T03:17:14,075 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39edb4ae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:17:14,076 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 394d2222f724,37299,-1 for getting cluster id 2024-11-10T03:17:14,076 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T03:17:14,077 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f5c6727-65f7-4659-afe8-8e9a55664c78' 2024-11-10T03:17:14,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T03:17:14,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f5c6727-65f7-4659-afe8-8e9a55664c78" 2024-11-10T03:17:14,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4baf45a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:17:14,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [394d2222f724,37299,-1] 2024-11-10T03:17:14,079 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T03:17:14,079 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:17:14,081 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36690, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T03:17:14,082 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21d0b352, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:17:14,082 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T03:17:14,083 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=394d2222f724,38139,1731208572054, seqNum=-1] 2024-11-10T03:17:14,084 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T03:17:14,085 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49910, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T03:17:14,085 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-10T03:17:14,086 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 394d2222f724,37299,1731208571229 2024-11-10T03:17:14,086 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@17824b2f 2024-11-10T03:17:14,086 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-10T03:17:14,087 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36692, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-10T03:17:14,088 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@167a19db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:17:14,088 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 394d2222f724,37299,-1 for getting cluster id 2024-11-10T03:17:14,088 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T03:17:14,089 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f5c6727-65f7-4659-afe8-8e9a55664c78' 2024-11-10T03:17:14,089 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T03:17:14,089 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f5c6727-65f7-4659-afe8-8e9a55664c78" 2024-11-10T03:17:14,089 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a4f8dc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:17:14,089 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [394d2222f724,37299,-1] 2024-11-10T03:17:14,089 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T03:17:14,090 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:17:14,090 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36708, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T03:17:14,091 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f05a07, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:17:14,091 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5be9e1d8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:17:14,091 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 394d2222f724,37299,-1 for getting cluster id 2024-11-10T03:17:14,091 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T03:17:14,092 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f5c6727-65f7-4659-afe8-8e9a55664c78' 2024-11-10T03:17:14,092 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T03:17:14,092 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f5c6727-65f7-4659-afe8-8e9a55664c78" 2024-11-10T03:17:14,092 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@685c3cb9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:17:14,092 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [394d2222f724,37299,-1] 2024-11-10T03:17:14,092 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T03:17:14,092 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:17:14,093 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36738, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T03:17:14,093 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@536432b2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:17:14,094 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e0c8c55, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:17:14,094 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 394d2222f724,37299,-1 for getting cluster id 2024-11-10T03:17:14,094 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T03:17:14,094 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f5c6727-65f7-4659-afe8-8e9a55664c78' 2024-11-10T03:17:14,094 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T03:17:14,095 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f5c6727-65f7-4659-afe8-8e9a55664c78" 2024-11-10T03:17:14,095 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ef76240, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:17:14,095 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [394d2222f724,37299,-1] 2024-11-10T03:17:14,095 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T03:17:14,095 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:17:14,095 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36758, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T03:17:14,096 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28b1d378, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:17:14,096 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a2ab9c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:17:14,096 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 394d2222f724,37299,-1 for getting cluster id 2024-11-10T03:17:14,096 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T03:17:14,097 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f5c6727-65f7-4659-afe8-8e9a55664c78' 2024-11-10T03:17:14,097 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T03:17:14,097 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f5c6727-65f7-4659-afe8-8e9a55664c78" 2024-11-10T03:17:14,098 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2452c29e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:17:14,098 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [394d2222f724,37299,-1] 2024-11-10T03:17:14,098 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T03:17:14,098 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:17:14,099 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36784, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T03:17:14,099 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46562e1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:17:14,100 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c19450f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:17:14,100 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 394d2222f724,37299,-1 for getting cluster id 2024-11-10T03:17:14,100 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T03:17:14,101 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f5c6727-65f7-4659-afe8-8e9a55664c78' 2024-11-10T03:17:14,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T03:17:14,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f5c6727-65f7-4659-afe8-8e9a55664c78" 2024-11-10T03:17:14,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7aa3bbbb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:17:14,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [394d2222f724,37299,-1] 2024-11-10T03:17:14,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T03:17:14,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:17:14,102 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36804, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T03:17:14,102 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b51bbe8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:17:14,103 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2715d903, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:17:14,103 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 394d2222f724,37299,-1 for getting cluster id 2024-11-10T03:17:14,103 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T03:17:14,104 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f5c6727-65f7-4659-afe8-8e9a55664c78' 2024-11-10T03:17:14,104 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T03:17:14,104 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f5c6727-65f7-4659-afe8-8e9a55664c78" 2024-11-10T03:17:14,104 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c80576c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:17:14,104 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [394d2222f724,37299,-1] 2024-11-10T03:17:14,104 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T03:17:14,104 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:17:14,105 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36816, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T03:17:14,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@154a7560, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:17:14,106 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@478d996b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:17:14,106 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 394d2222f724,37299,-1 for getting cluster id 2024-11-10T03:17:14,106 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T03:17:14,107 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f5c6727-65f7-4659-afe8-8e9a55664c78' 2024-11-10T03:17:14,107 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T03:17:14,107 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f5c6727-65f7-4659-afe8-8e9a55664c78" 2024-11-10T03:17:14,107 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d9292f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:17:14,107 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [394d2222f724,37299,-1] 2024-11-10T03:17:14,107 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T03:17:14,108 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:17:14,108 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36844, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T03:17:14,109 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f970279, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:17:14,109 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ee56858, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:17:14,109 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 394d2222f724,37299,-1 for getting cluster id 2024-11-10T03:17:14,109 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T03:17:14,110 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f5c6727-65f7-4659-afe8-8e9a55664c78' 2024-11-10T03:17:14,110 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T03:17:14,110 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f5c6727-65f7-4659-afe8-8e9a55664c78" 2024-11-10T03:17:14,110 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21968ee0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:17:14,110 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [394d2222f724,37299,-1] 2024-11-10T03:17:14,111 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T03:17:14,111 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:17:14,111 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36866, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T03:17:14,112 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73a9df9d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:17:14,112 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56924c9a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:17:14,112 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 394d2222f724,37299,-1 for getting cluster id 2024-11-10T03:17:14,113 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T03:17:14,113 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f5c6727-65f7-4659-afe8-8e9a55664c78' 2024-11-10T03:17:14,113 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T03:17:14,113 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f5c6727-65f7-4659-afe8-8e9a55664c78" 2024-11-10T03:17:14,113 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c76ddb3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:17:14,114 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [394d2222f724,37299,-1] 2024-11-10T03:17:14,114 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T03:17:14,114 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:17:14,114 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36886, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T03:17:14,115 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43c5c2dc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:17:14,115 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10670012, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:17:14,115 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 394d2222f724,37299,-1 for getting cluster id 2024-11-10T03:17:14,115 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T03:17:14,116 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f5c6727-65f7-4659-afe8-8e9a55664c78' 2024-11-10T03:17:14,116 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T03:17:14,116 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f5c6727-65f7-4659-afe8-8e9a55664c78" 2024-11-10T03:17:14,116 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b98a4f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:17:14,116 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [394d2222f724,37299,-1] 2024-11-10T03:17:14,116 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T03:17:14,116 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:17:14,117 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36916, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T03:17:14,117 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27fe39e5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:17:14,118 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T03:17:14,118 DEBUG [Thread-1148 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T03:17:14,118 DEBUG [Thread-1131 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T03:17:14,119 DEBUG [Thread-1137 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T03:17:14,119 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=394d2222f724,38139,1731208572054, seqNum=-1] 2024-11-10T03:17:14,119 DEBUG [Thread-1135 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T03:17:14,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees 2024-11-10T03:17:14,119 DEBUG [Thread-1140 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T03:17:14,119 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T03:17:14,119 DEBUG [Thread-1142 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T03:17:14,119 DEBUG [Thread-1146 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T03:17:14,120 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=394d2222f724,38139,1731208572054, seqNum=-1] 2024-11-10T03:17:14,120 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=394d2222f724,38139,1731208572054, seqNum=-1] 2024-11-10T03:17:14,120 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=394d2222f724,38139,1731208572054, seqNum=-1] 2024-11-10T03:17:14,120 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T03:17:14,120 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T03:17:14,120 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T03:17:14,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-10T03:17:14,120 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49926, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T03:17:14,121 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=394d2222f724,38139,1731208572054, seqNum=-1] 2024-11-10T03:17:14,121 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T03:17:14,121 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49950, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T03:17:14,121 DEBUG [Thread-1140 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=394d2222f724,38139,1731208572054, seqNum=-1] 2024-11-10T03:17:14,122 DEBUG [Thread-1140 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T03:17:14,122 DEBUG [Thread-1129 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T03:17:14,122 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49966, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T03:17:14,122 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=394d2222f724,38139,1731208572054, seqNum=-1] 2024-11-10T03:17:14,122 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T03:17:14,123 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49972, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T03:17:14,123 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49936, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T03:17:14,123 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49970, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T03:17:14,123 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2] 2024-11-10T03:17:14,123 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=394d2222f724,38139,1731208572054, seqNum=-1] 2024-11-10T03:17:14,123 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2] 2024-11-10T03:17:14,123 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T03:17:14,125 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49990, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T03:17:14,125 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2] 2024-11-10T03:17:14,125 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2] 2024-11-10T03:17:14,125 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49974, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T03:17:14,125 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2] 2024-11-10T03:17:14,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2] 2024-11-10T03:17:14,127 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T03:17:14,128 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2] 2024-11-10T03:17:14,128 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2] 2024-11-10T03:17:14,128 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T03:17:14,128 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T03:17:14,137 DEBUG [Thread-1133 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T03:17:14,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on a93817da82260912633bd4e3d359d5b7 2024-11-10T03:17:14,137 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a93817da82260912633bd4e3d359d5b7 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-10T03:17:14,137 DEBUG [Thread-1144 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T03:17:14,138 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=394d2222f724,38139,1731208572054, seqNum=-1] 2024-11-10T03:17:14,138 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T03:17:14,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a93817da82260912633bd4e3d359d5b7, store=A 2024-11-10T03:17:14,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:17:14,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a93817da82260912633bd4e3d359d5b7, store=B 2024-11-10T03:17:14,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:17:14,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a93817da82260912633bd4e3d359d5b7, store=C 2024-11-10T03:17:14,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:17:14,141 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50002, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T03:17:14,145 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=394d2222f724,38139,1731208572054, seqNum=-1] 2024-11-10T03:17:14,145 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T03:17:14,146 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2] 2024-11-10T03:17:14,153 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50004, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T03:17:14,166 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2] 2024-11-10T03:17:14,182 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/A/5d0cd48ca17f445a9c9fb66413b05919 is 50, key is test_row_0/A:col10/1731208634137/Put/seqid=0 2024-11-10T03:17:14,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742068_1244 (size=12001) 2024-11-10T03:17:14,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-10T03:17:14,228 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731208634177, max timestamp=1731208634221Num uniques 859; 2024-11-10T03:17:14,228 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731208634177, max timestamp=1731208634221Num uniques 859; 2024-11-10T03:17:14,228 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731208634177, max timestamp=1731208634221Num uniques 859; 2024-11-10T03:17:14,228 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T03:17:14,228 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T03:17:14,228 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T03:17:14,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:14,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49936 deadline: 1731208694258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:14,268 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:14,268 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:14,268 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:14,274 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:14,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:14,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50002 deadline: 1731208694267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:14,276 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:14,276 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:14,276 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:14,276 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:14,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49990 deadline: 1731208694267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:14,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:14,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:14,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:14,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:14,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49950 deadline: 1731208694268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:14,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49966 deadline: 1731208694264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:14,277 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:14,277 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:14,277 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:14,278 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:14,278 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:14,278 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:14,282 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-10T03:17:14,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:14,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. as already flushing 2024-11-10T03:17:14,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:14,283 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:14,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:14,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:14,377 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:14,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49936 deadline: 1731208694373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:14,378 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:14,378 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:14,378 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:14,385 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:14,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49990 deadline: 1731208694383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:14,386 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:14,386 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:14,386 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:14,386 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:14,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49966 deadline: 1731208694383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:14,386 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:14,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49950 deadline: 1731208694383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:14,386 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:14,386 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:14,386 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:14,387 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:14,387 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:14,387 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:14,387 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:14,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50002 deadline: 1731208694383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:14,389 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:14,389 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:14,389 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:14,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-10T03:17:14,444 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-10T03:17:14,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:14,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. as already flushing 2024-11-10T03:17:14,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:14,445 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:14,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:14,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:14,588 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:14,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49936 deadline: 1731208694583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:14,588 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:14,588 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:14,588 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:14,592 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/A/5d0cd48ca17f445a9c9fb66413b05919 2024-11-10T03:17:14,596 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:14,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49990 deadline: 1731208694593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:14,596 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:14,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49950 deadline: 1731208694593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:14,597 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:14,597 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:14,597 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:14,597 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:14,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50002 deadline: 1731208694593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:14,597 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:14,597 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:14,597 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:14,597 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:14,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49966 deadline: 1731208694593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:14,598 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:14,598 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:14,598 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:14,600 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:14,600 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:14,600 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:14,602 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-10T03:17:14,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:14,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. as already flushing 2024-11-10T03:17:14,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:14,603 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:14,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:14,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:14,635 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/B/3033361a4f5841fea05de4f7ee541ad3 is 50, key is test_row_0/B:col10/1731208634137/Put/seqid=0 2024-11-10T03:17:14,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742069_1245 (size=12001) 2024-11-10T03:17:14,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-10T03:17:14,755 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-10T03:17:14,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:14,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. as already flushing 2024-11-10T03:17:14,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:14,756 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:14,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:14,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:14,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:14,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49936 deadline: 1731208694893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:14,896 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:14,896 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:14,896 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:14,906 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:14,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49966 deadline: 1731208694903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:14,906 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:14,907 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:14,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49990 deadline: 1731208694903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:14,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49950 deadline: 1731208694904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:14,907 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:14,907 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:14,907 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:14,907 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:14,907 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:14,907 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:14,907 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:14,907 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:14,907 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:14,907 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:14,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50002 deadline: 1731208694904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:14,907 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:14,907 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:14,907 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:14,909 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-10T03:17:14,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:14,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. as already flushing 2024-11-10T03:17:14,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:14,909 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:14,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:14,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:15,062 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/B/3033361a4f5841fea05de4f7ee541ad3 2024-11-10T03:17:15,067 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-10T03:17:15,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:15,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. as already flushing 2024-11-10T03:17:15,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:15,068 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:15,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:15,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:15,091 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/C/da2f8bdc98d44f989ebe19e1835f3858 is 50, key is test_row_0/C:col10/1731208634137/Put/seqid=0 2024-11-10T03:17:15,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742070_1246 (size=12001) 2024-11-10T03:17:15,221 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-10T03:17:15,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:15,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. as already flushing 2024-11-10T03:17:15,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:15,221 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:15,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:15,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:15,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-10T03:17:15,373 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-10T03:17:15,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:15,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. as already flushing 2024-11-10T03:17:15,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:15,374 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:15,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:15,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:15,405 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:15,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49936 deadline: 1731208695402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:15,406 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:15,406 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:15,406 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:15,416 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:15,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49990 deadline: 1731208695413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:15,417 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:15,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49950 deadline: 1731208695413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:15,417 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:15,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49966 deadline: 1731208695414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:15,418 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:15,418 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:15,418 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:15,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50002 deadline: 1731208695414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:15,418 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:15,420 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:15,420 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:15,421 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:15,421 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:15,421 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:15,421 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:15,421 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:15,421 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:15,421 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:15,513 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/C/da2f8bdc98d44f989ebe19e1835f3858 2024-11-10T03:17:15,528 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-10T03:17:15,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:15,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. as already flushing 2024-11-10T03:17:15,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:15,529 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:15,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:15,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:15,536 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/A/5d0cd48ca17f445a9c9fb66413b05919 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/5d0cd48ca17f445a9c9fb66413b05919 2024-11-10T03:17:15,551 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/5d0cd48ca17f445a9c9fb66413b05919, entries=150, sequenceid=12, filesize=11.7 K 2024-11-10T03:17:15,553 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/B/3033361a4f5841fea05de4f7ee541ad3 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/3033361a4f5841fea05de4f7ee541ad3 2024-11-10T03:17:15,565 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/3033361a4f5841fea05de4f7ee541ad3, entries=150, sequenceid=12, filesize=11.7 K 2024-11-10T03:17:15,574 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/C/da2f8bdc98d44f989ebe19e1835f3858 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/da2f8bdc98d44f989ebe19e1835f3858 2024-11-10T03:17:15,585 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/da2f8bdc98d44f989ebe19e1835f3858, entries=150, sequenceid=12, filesize=11.7 K 2024-11-10T03:17:15,586 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for a93817da82260912633bd4e3d359d5b7 in 1449ms, sequenceid=12, compaction requested=false 2024-11-10T03:17:15,586 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a93817da82260912633bd4e3d359d5b7: 2024-11-10T03:17:15,688 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-10T03:17:15,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:15,689 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2902): Flushing a93817da82260912633bd4e3d359d5b7 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-11-10T03:17:15,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a93817da82260912633bd4e3d359d5b7, store=A 2024-11-10T03:17:15,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T03:17:15,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a93817da82260912633bd4e3d359d5b7, store=B 2024-11-10T03:17:15,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T03:17:15,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a93817da82260912633bd4e3d359d5b7, store=C 2024-11-10T03:17:15,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T03:17:15,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/A/8c208cb4ecd946c296ba5ee2b5da7075 is 50, key is test_row_0/A:col10/1731208634266/Put/seqid=0 2024-11-10T03:17:15,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742071_1247 (size=12001) 2024-11-10T03:17:16,143 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/A/8c208cb4ecd946c296ba5ee2b5da7075 2024-11-10T03:17:16,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/B/977b9366611f49269f5f2ec72ff1ea7d is 50, key is test_row_0/B:col10/1731208634266/Put/seqid=0 2024-11-10T03:17:16,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742072_1248 (size=12001) 2024-11-10T03:17:16,174 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/B/977b9366611f49269f5f2ec72ff1ea7d 2024-11-10T03:17:16,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/C/5dc01dcc13a34c879adabaf80c5e2fe7 is 50, key is test_row_0/C:col10/1731208634266/Put/seqid=0 2024-11-10T03:17:16,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742073_1249 (size=12001) 2024-11-10T03:17:16,216 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/C/5dc01dcc13a34c879adabaf80c5e2fe7 2024-11-10T03:17:16,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/A/8c208cb4ecd946c296ba5ee2b5da7075 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/8c208cb4ecd946c296ba5ee2b5da7075 2024-11-10T03:17:16,241 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/8c208cb4ecd946c296ba5ee2b5da7075, entries=150, sequenceid=48, filesize=11.7 K 2024-11-10T03:17:16,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/B/977b9366611f49269f5f2ec72ff1ea7d as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/977b9366611f49269f5f2ec72ff1ea7d 2024-11-10T03:17:16,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-10T03:17:16,270 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/977b9366611f49269f5f2ec72ff1ea7d, entries=150, sequenceid=48, filesize=11.7 K 2024-11-10T03:17:16,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/C/5dc01dcc13a34c879adabaf80c5e2fe7 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/5dc01dcc13a34c879adabaf80c5e2fe7 2024-11-10T03:17:16,293 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/5dc01dcc13a34c879adabaf80c5e2fe7, entries=150, sequenceid=48, filesize=11.7 K 2024-11-10T03:17:16,296 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=0 B/0 for a93817da82260912633bd4e3d359d5b7 in 608ms, sequenceid=48, compaction requested=false 2024-11-10T03:17:16,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2603): Flush status journal for a93817da82260912633bd4e3d359d5b7: 2024-11-10T03:17:16,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:16,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-11-10T03:17:16,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-11-10T03:17:16,301 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-10T03:17:16,301 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1700 sec 2024-11-10T03:17:16,305 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees in 2.1840 sec 2024-11-10T03:17:16,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on a93817da82260912633bd4e3d359d5b7 2024-11-10T03:17:16,437 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a93817da82260912633bd4e3d359d5b7 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-10T03:17:16,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a93817da82260912633bd4e3d359d5b7, store=A 2024-11-10T03:17:16,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:17:16,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a93817da82260912633bd4e3d359d5b7, store=B 2024-11-10T03:17:16,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:17:16,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a93817da82260912633bd4e3d359d5b7, store=C 2024-11-10T03:17:16,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:17:16,462 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/A/406dcc221b2240d4a139bbde039adaa1 is 50, key is test_row_0/A:col10/1731208636437/Put/seqid=0 2024-11-10T03:17:16,480 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:16,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49990 deadline: 1731208696467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:16,481 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:16,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:16,481 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:16,481 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:16,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49936 deadline: 1731208696470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:16,481 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:16,481 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:16,481 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:16,482 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731208636460, max timestamp=1731208636478Num uniques 859; 2024-11-10T03:17:16,482 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-10T03:17:16,486 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731208636467, max timestamp=1731208636478Num uniques 859; 2024-11-10T03:17:16,486 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-10T03:17:16,489 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731208636460, max timestamp=1731208636478Num uniques 859; 2024-11-10T03:17:16,489 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T03:17:16,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742074_1250 (size=14341) 2024-11-10T03:17:16,498 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/A/406dcc221b2240d4a139bbde039adaa1 2024-11-10T03:17:16,509 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/B/5740a8e4cd344af28d5326f61f530351 is 50, key is test_row_0/B:col10/1731208636437/Put/seqid=0 2024-11-10T03:17:16,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742075_1251 (size=12001) 2024-11-10T03:17:16,562 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:16,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49950 deadline: 1731208696557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:16,563 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:16,563 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:16,563 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:16,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:16,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50002 deadline: 1731208696563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:16,571 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:16,571 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:16,571 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:16,571 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:16,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49966 deadline: 1731208696563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:16,577 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:16,577 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:16,577 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:16,586 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:16,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49936 deadline: 1731208696582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:16,591 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:16,591 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:16,591 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:16,597 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:16,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49990 deadline: 1731208696593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:16,597 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:16,597 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:16,597 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:16,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:16,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49950 deadline: 1731208696672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:16,677 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:16,677 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:16,677 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:16,686 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:16,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50002 deadline: 1731208696683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:16,687 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:16,687 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:16,687 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:16,687 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:16,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49966 deadline: 1731208696683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:16,687 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:16,687 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:16,687 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:16,795 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:16,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49936 deadline: 1731208696793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:16,795 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:16,795 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:16,795 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:16,805 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:16,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49990 deadline: 1731208696803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:16,805 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:16,805 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:16,805 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:16,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:16,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49950 deadline: 1731208696883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:16,886 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:16,886 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:16,886 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:16,896 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:16,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50002 deadline: 1731208696893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:16,897 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:16,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49966 deadline: 1731208696893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:16,897 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:16,897 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:16,897 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:16,897 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:16,897 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:16,897 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:16,918 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/B/5740a8e4cd344af28d5326f61f530351 2024-11-10T03:17:16,927 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/C/c5c7724d0de94fcf98acabc47b2440e5 is 50, key is test_row_0/C:col10/1731208636437/Put/seqid=0 2024-11-10T03:17:16,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742076_1252 (size=12001) 2024-11-10T03:17:16,961 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/C/c5c7724d0de94fcf98acabc47b2440e5 2024-11-10T03:17:16,967 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/A/406dcc221b2240d4a139bbde039adaa1 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/406dcc221b2240d4a139bbde039adaa1 2024-11-10T03:17:16,973 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/406dcc221b2240d4a139bbde039adaa1, entries=200, sequenceid=59, filesize=14.0 K 2024-11-10T03:17:16,974 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/B/5740a8e4cd344af28d5326f61f530351 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/5740a8e4cd344af28d5326f61f530351 2024-11-10T03:17:16,979 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/5740a8e4cd344af28d5326f61f530351, entries=150, sequenceid=59, filesize=11.7 K 2024-11-10T03:17:16,980 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/C/c5c7724d0de94fcf98acabc47b2440e5 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/c5c7724d0de94fcf98acabc47b2440e5 2024-11-10T03:17:16,992 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/c5c7724d0de94fcf98acabc47b2440e5, entries=150, sequenceid=59, filesize=11.7 K 2024-11-10T03:17:16,993 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=133.78 KB/136995 for a93817da82260912633bd4e3d359d5b7 in 557ms, sequenceid=59, compaction requested=true 2024-11-10T03:17:16,993 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a93817da82260912633bd4e3d359d5b7: 2024-11-10T03:17:16,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a93817da82260912633bd4e3d359d5b7:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T03:17:16,993 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T03:17:16,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:17:16,993 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T03:17:16,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a93817da82260912633bd4e3d359d5b7:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T03:17:16,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:17:16,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a93817da82260912633bd4e3d359d5b7:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T03:17:16,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T03:17:16,994 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T03:17:16,995 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1541): a93817da82260912633bd4e3d359d5b7/A is initiating minor compaction (all files) 2024-11-10T03:17:16,995 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a93817da82260912633bd4e3d359d5b7/A in TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:16,995 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/5d0cd48ca17f445a9c9fb66413b05919, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/8c208cb4ecd946c296ba5ee2b5da7075, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/406dcc221b2240d4a139bbde039adaa1] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp, totalSize=37.4 K 2024-11-10T03:17:16,995 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T03:17:16,995 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1541): a93817da82260912633bd4e3d359d5b7/B is initiating minor compaction (all files) 2024-11-10T03:17:16,995 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a93817da82260912633bd4e3d359d5b7/B in TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:16,995 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/3033361a4f5841fea05de4f7ee541ad3, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/977b9366611f49269f5f2ec72ff1ea7d, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/5740a8e4cd344af28d5326f61f530351] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp, totalSize=35.2 K 2024-11-10T03:17:16,995 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5d0cd48ca17f445a9c9fb66413b05919, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731208634124 2024-11-10T03:17:16,996 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8c208cb4ecd946c296ba5ee2b5da7075, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731208634261 2024-11-10T03:17:16,996 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 3033361a4f5841fea05de4f7ee541ad3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731208634124 2024-11-10T03:17:16,996 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 406dcc221b2240d4a139bbde039adaa1, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731208636426 2024-11-10T03:17:16,997 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 977b9366611f49269f5f2ec72ff1ea7d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731208634261 2024-11-10T03:17:16,997 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 5740a8e4cd344af28d5326f61f530351, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731208636434 2024-11-10T03:17:17,016 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a93817da82260912633bd4e3d359d5b7#A#compaction#204 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:17:17,018 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/A/f160c455621c4049b83435a88ab226c3 is 50, key is test_row_0/A:col10/1731208636437/Put/seqid=0 2024-11-10T03:17:17,022 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a93817da82260912633bd4e3d359d5b7#B#compaction#205 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:17:17,023 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/B/7bd58034c1164a9a98e73981239ba58a is 50, key is test_row_0/B:col10/1731208636437/Put/seqid=0 2024-11-10T03:17:17,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742077_1253 (size=12104) 2024-11-10T03:17:17,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742078_1254 (size=12104) 2024-11-10T03:17:17,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on a93817da82260912633bd4e3d359d5b7 2024-11-10T03:17:17,106 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a93817da82260912633bd4e3d359d5b7 3/3 column families, dataSize=140.49 KB heapSize=399.30 KB 2024-11-10T03:17:17,107 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a93817da82260912633bd4e3d359d5b7, store=A 2024-11-10T03:17:17,107 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T03:17:17,107 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a93817da82260912633bd4e3d359d5b7, store=B 2024-11-10T03:17:17,107 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T03:17:17,107 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a93817da82260912633bd4e3d359d5b7, store=C 2024-11-10T03:17:17,107 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T03:17:17,125 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-10T03:17:17,131 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/A/79d58dd597a94b158913873cc60a55ba is 50, key is test_row_0/A:col10/1731208636555/Put/seqid=0 2024-11-10T03:17:17,148 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:17,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49936 deadline: 1731208697143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:17,149 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:17,149 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:17,149 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:17,157 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:17,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49990 deadline: 1731208697150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:17,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742079_1255 (size=14341) 2024-11-10T03:17:17,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:17,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:17,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:17,160 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/A/79d58dd597a94b158913873cc60a55ba 2024-11-10T03:17:17,197 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:17,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49950 deadline: 1731208697193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:17,198 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:17,198 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:17,198 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:17,206 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/B/d4b9f0d0da824cc2baf171c2e3f5177d is 50, key is test_row_0/B:col10/1731208636555/Put/seqid=0 2024-11-10T03:17:17,207 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:17,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49966 deadline: 1731208697203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:17,207 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:17,207 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:17,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50002 deadline: 1731208697203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:17,207 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:17,207 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:17,208 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:17,208 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:17,208 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:17,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742080_1256 (size=12001) 2024-11-10T03:17:17,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:17,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49936 deadline: 1731208697257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:17,259 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:17,259 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:17,259 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:17,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:17,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49990 deadline: 1731208697268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:17,280 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:17,280 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:17,280 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:17,451 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/A/f160c455621c4049b83435a88ab226c3 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/f160c455621c4049b83435a88ab226c3 2024-11-10T03:17:17,457 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a93817da82260912633bd4e3d359d5b7/A of a93817da82260912633bd4e3d359d5b7 into f160c455621c4049b83435a88ab226c3(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:17:17,457 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a93817da82260912633bd4e3d359d5b7: 2024-11-10T03:17:17,457 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., storeName=a93817da82260912633bd4e3d359d5b7/A, priority=13, startTime=1731208636993; duration=0sec 2024-11-10T03:17:17,457 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T03:17:17,457 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a93817da82260912633bd4e3d359d5b7:A 2024-11-10T03:17:17,457 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T03:17:17,458 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/B/7bd58034c1164a9a98e73981239ba58a as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/7bd58034c1164a9a98e73981239ba58a 2024-11-10T03:17:17,460 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T03:17:17,460 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1541): a93817da82260912633bd4e3d359d5b7/C is initiating minor compaction (all files) 2024-11-10T03:17:17,460 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a93817da82260912633bd4e3d359d5b7/C in TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:17,461 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/da2f8bdc98d44f989ebe19e1835f3858, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/5dc01dcc13a34c879adabaf80c5e2fe7, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/c5c7724d0de94fcf98acabc47b2440e5] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp, totalSize=35.2 K 2024-11-10T03:17:17,461 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting da2f8bdc98d44f989ebe19e1835f3858, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731208634124 2024-11-10T03:17:17,462 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5dc01dcc13a34c879adabaf80c5e2fe7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731208634261 2024-11-10T03:17:17,463 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting c5c7724d0de94fcf98acabc47b2440e5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731208636434 2024-11-10T03:17:17,465 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:17,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49936 deadline: 1731208697463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:17,465 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:17,465 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:17,465 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:17,467 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a93817da82260912633bd4e3d359d5b7/B of a93817da82260912633bd4e3d359d5b7 into 7bd58034c1164a9a98e73981239ba58a(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:17:17,467 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a93817da82260912633bd4e3d359d5b7: 2024-11-10T03:17:17,467 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., storeName=a93817da82260912633bd4e3d359d5b7/B, priority=13, startTime=1731208636993; duration=0sec 2024-11-10T03:17:17,467 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:17:17,467 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a93817da82260912633bd4e3d359d5b7:B 2024-11-10T03:17:17,478 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a93817da82260912633bd4e3d359d5b7#C#compaction#208 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:17:17,478 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/C/20ff6cff7e744e5d8e83553111e4005c is 50, key is test_row_0/C:col10/1731208636437/Put/seqid=0 2024-11-10T03:17:17,499 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:17,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49990 deadline: 1731208697494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:17,499 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:17,499 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:17,499 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:17,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742081_1257 (size=12104) 2024-11-10T03:17:17,520 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/C/20ff6cff7e744e5d8e83553111e4005c as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/20ff6cff7e744e5d8e83553111e4005c 2024-11-10T03:17:17,527 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a93817da82260912633bd4e3d359d5b7/C of a93817da82260912633bd4e3d359d5b7 into 20ff6cff7e744e5d8e83553111e4005c(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:17:17,527 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a93817da82260912633bd4e3d359d5b7: 2024-11-10T03:17:17,527 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., storeName=a93817da82260912633bd4e3d359d5b7/C, priority=13, startTime=1731208636993; duration=0sec 2024-11-10T03:17:17,527 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:17:17,527 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a93817da82260912633bd4e3d359d5b7:C 2024-11-10T03:17:17,649 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/B/d4b9f0d0da824cc2baf171c2e3f5177d 2024-11-10T03:17:17,658 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/C/cd5d734f48fa4d4aad7563ef742b0d43 is 50, key is test_row_0/C:col10/1731208636555/Put/seqid=0 2024-11-10T03:17:17,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742082_1258 (size=12001) 2024-11-10T03:17:17,705 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:17,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49950 deadline: 1731208697702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:17,706 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:17,706 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:17,706 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:17,717 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:17,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49966 deadline: 1731208697713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:17,717 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:17,717 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:17,717 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:17,726 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:17,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50002 deadline: 1731208697723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:17,727 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:17,727 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:17,727 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:17,776 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:17,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49936 deadline: 1731208697772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:17,776 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:17,776 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:17,776 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:17,804 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:17,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49990 deadline: 1731208697803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:17,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:17,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:17,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:18,065 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.33 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/C/cd5d734f48fa4d4aad7563ef742b0d43 2024-11-10T03:17:18,069 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/A/79d58dd597a94b158913873cc60a55ba as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/79d58dd597a94b158913873cc60a55ba 2024-11-10T03:17:18,073 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/79d58dd597a94b158913873cc60a55ba, entries=200, sequenceid=96, filesize=14.0 K 2024-11-10T03:17:18,074 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/B/d4b9f0d0da824cc2baf171c2e3f5177d as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/d4b9f0d0da824cc2baf171c2e3f5177d 2024-11-10T03:17:18,080 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/d4b9f0d0da824cc2baf171c2e3f5177d, entries=150, sequenceid=96, filesize=11.7 K 2024-11-10T03:17:18,083 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/C/cd5d734f48fa4d4aad7563ef742b0d43 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/cd5d734f48fa4d4aad7563ef742b0d43 2024-11-10T03:17:18,089 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/cd5d734f48fa4d4aad7563ef742b0d43, entries=150, sequenceid=96, filesize=11.7 K 2024-11-10T03:17:18,090 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.49 KB/143865, heapSize ~399.26 KB/408840, currentSize=40.25 KB/41220 for a93817da82260912633bd4e3d359d5b7 in 984ms, sequenceid=96, compaction requested=false 2024-11-10T03:17:18,090 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a93817da82260912633bd4e3d359d5b7: 2024-11-10T03:17:18,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-10T03:17:18,284 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T03:17:18,284 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T03:17:18,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-11-10T03:17:18,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on a93817da82260912633bd4e3d359d5b7 2024-11-10T03:17:18,286 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a93817da82260912633bd4e3d359d5b7 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-10T03:17:18,286 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a93817da82260912633bd4e3d359d5b7, store=A 2024-11-10T03:17:18,286 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:17:18,286 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a93817da82260912633bd4e3d359d5b7, store=B 2024-11-10T03:17:18,286 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:17:18,286 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a93817da82260912633bd4e3d359d5b7, store=C 2024-11-10T03:17:18,286 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:17:18,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-10T03:17:18,288 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T03:17:18,290 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T03:17:18,290 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T03:17:18,296 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/A/0325c28dd6794bbaaa29dc9378d977cb is 50, key is test_row_0/A:col10/1731208637140/Put/seqid=0 2024-11-10T03:17:18,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742083_1259 (size=12001) 2024-11-10T03:17:18,316 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=109 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/A/0325c28dd6794bbaaa29dc9378d977cb 2024-11-10T03:17:18,326 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/B/1c53949971274fd8aa5001bf5d6b477a is 50, key is test_row_0/B:col10/1731208637140/Put/seqid=0 2024-11-10T03:17:18,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742084_1260 (size=12001) 2024-11-10T03:17:18,365 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731208638321, max timestamp=1731208638352Num uniques 859; 2024-11-10T03:17:18,366 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T03:17:18,366 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731208638321, max timestamp=1731208638352Num uniques 859; 2024-11-10T03:17:18,366 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T03:17:18,368 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731208638321, max timestamp=1731208638352Num uniques 859; 2024-11-10T03:17:18,368 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T03:17:18,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-10T03:17:18,433 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:18,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49936 deadline: 1731208698428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:18,433 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:18,433 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:18,433 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:18,434 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:18,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49990 deadline: 1731208698431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:18,435 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:18,435 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:18,435 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:18,443 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-10T03:17:18,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:18,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. as already flushing 2024-11-10T03:17:18,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:18,443 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:18,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:18,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:18,544 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:18,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49936 deadline: 1731208698542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:18,544 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:18,545 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:18,545 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:18,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49990 deadline: 1731208698542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:18,545 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:18,545 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:18,545 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:18,545 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:18,596 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-10T03:17:18,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:18,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. as already flushing 2024-11-10T03:17:18,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:18,596 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:18,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:18,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:18,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-10T03:17:18,714 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:18,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49950 deadline: 1731208698713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:18,715 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 394d2222f724,38139,1731208572054 for 'test_row_0' in a93817da82260912633bd4e3d359d5b7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2158 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T03:17:18,716 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:18,716 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:18,716 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:18,725 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:18,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49966 deadline: 1731208698723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:18,725 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 394d2222f724,38139,1731208572054 for 'test_row_2' in a93817da82260912633bd4e3d359d5b7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T03:17:18,726 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:18,726 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:18,726 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:18,740 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=109 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/B/1c53949971274fd8aa5001bf5d6b477a 2024-11-10T03:17:18,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:18,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50002 deadline: 1731208698743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:18,745 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 394d2222f724,38139,1731208572054 for 'test_row_2' in a93817da82260912633bd4e3d359d5b7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T03:17:18,745 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:18,745 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:18,745 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:18,747 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/C/2adfe9d89dfb4b02ad904ef6c0c3105a is 50, key is test_row_0/C:col10/1731208637140/Put/seqid=0 2024-11-10T03:17:18,751 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-10T03:17:18,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:18,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. as already flushing 2024-11-10T03:17:18,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:18,751 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:18,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:18,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:18,757 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:18,757 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:18,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49936 deadline: 1731208698753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:18,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49990 deadline: 1731208698753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:18,758 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:18,758 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:18,758 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:18,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:18,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:18,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:18,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742085_1261 (size=12001) 2024-11-10T03:17:18,904 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-10T03:17:18,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:18,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. as already flushing 2024-11-10T03:17:18,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:18,905 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:18,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:18,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:18,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-10T03:17:19,057 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-10T03:17:19,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:19,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. as already flushing 2024-11-10T03:17:19,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:19,058 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:19,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:19,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:19,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:19,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:19,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49990 deadline: 1731208699063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:19,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49936 deadline: 1731208699063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:19,067 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:19,067 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:19,067 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:19,076 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:19,076 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:19,076 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:19,153 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=109 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/C/2adfe9d89dfb4b02ad904ef6c0c3105a 2024-11-10T03:17:19,159 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/A/0325c28dd6794bbaaa29dc9378d977cb as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/0325c28dd6794bbaaa29dc9378d977cb 2024-11-10T03:17:19,164 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/0325c28dd6794bbaaa29dc9378d977cb, entries=150, sequenceid=109, filesize=11.7 K 2024-11-10T03:17:19,165 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/B/1c53949971274fd8aa5001bf5d6b477a as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/1c53949971274fd8aa5001bf5d6b477a 2024-11-10T03:17:19,171 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/1c53949971274fd8aa5001bf5d6b477a, entries=150, sequenceid=109, filesize=11.7 K 2024-11-10T03:17:19,172 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/C/2adfe9d89dfb4b02ad904ef6c0c3105a as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/2adfe9d89dfb4b02ad904ef6c0c3105a 2024-11-10T03:17:19,178 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/2adfe9d89dfb4b02ad904ef6c0c3105a, entries=150, sequenceid=109, filesize=11.7 K 2024-11-10T03:17:19,179 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for a93817da82260912633bd4e3d359d5b7 in 894ms, sequenceid=109, compaction requested=true 2024-11-10T03:17:19,179 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a93817da82260912633bd4e3d359d5b7: 2024-11-10T03:17:19,179 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a93817da82260912633bd4e3d359d5b7:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T03:17:19,179 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:17:19,179 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T03:17:19,179 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a93817da82260912633bd4e3d359d5b7:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T03:17:19,179 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:17:19,179 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T03:17:19,179 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a93817da82260912633bd4e3d359d5b7:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T03:17:19,179 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T03:17:19,181 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T03:17:19,181 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T03:17:19,181 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1541): a93817da82260912633bd4e3d359d5b7/B is initiating minor compaction (all files) 2024-11-10T03:17:19,181 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1541): a93817da82260912633bd4e3d359d5b7/A is initiating minor compaction (all files) 2024-11-10T03:17:19,181 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a93817da82260912633bd4e3d359d5b7/B in TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:19,181 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a93817da82260912633bd4e3d359d5b7/A in TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:19,182 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/7bd58034c1164a9a98e73981239ba58a, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/d4b9f0d0da824cc2baf171c2e3f5177d, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/1c53949971274fd8aa5001bf5d6b477a] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp, totalSize=35.3 K 2024-11-10T03:17:19,182 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/f160c455621c4049b83435a88ab226c3, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/79d58dd597a94b158913873cc60a55ba, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/0325c28dd6794bbaaa29dc9378d977cb] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp, totalSize=37.5 K 2024-11-10T03:17:19,182 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 7bd58034c1164a9a98e73981239ba58a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731208636434 2024-11-10T03:17:19,182 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting f160c455621c4049b83435a88ab226c3, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731208636434 2024-11-10T03:17:19,182 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 79d58dd597a94b158913873cc60a55ba, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731208636552 2024-11-10T03:17:19,182 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting d4b9f0d0da824cc2baf171c2e3f5177d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731208636552 2024-11-10T03:17:19,184 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0325c28dd6794bbaaa29dc9378d977cb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1731208637140 2024-11-10T03:17:19,185 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c53949971274fd8aa5001bf5d6b477a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1731208637140 2024-11-10T03:17:19,210 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-10T03:17:19,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:19,211 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing a93817da82260912633bd4e3d359d5b7 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-10T03:17:19,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a93817da82260912633bd4e3d359d5b7, store=A 2024-11-10T03:17:19,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T03:17:19,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a93817da82260912633bd4e3d359d5b7, store=B 2024-11-10T03:17:19,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T03:17:19,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a93817da82260912633bd4e3d359d5b7, store=C 2024-11-10T03:17:19,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T03:17:19,216 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a93817da82260912633bd4e3d359d5b7#A#compaction#213 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:17:19,217 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/A/2c739a58bc5e4997aaf9895d2e0e5110 is 50, key is test_row_0/A:col10/1731208637140/Put/seqid=0 2024-11-10T03:17:19,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/A/f5e46cc015074f98b9a17c877e9662c7 is 50, key is test_row_0/A:col10/1731208638414/Put/seqid=0 2024-11-10T03:17:19,227 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a93817da82260912633bd4e3d359d5b7#B#compaction#215 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:17:19,228 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/B/161433a883984db6964b963faaaa5197 is 50, key is test_row_0/B:col10/1731208637140/Put/seqid=0 2024-11-10T03:17:19,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742086_1262 (size=12207) 2024-11-10T03:17:19,279 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/A/2c739a58bc5e4997aaf9895d2e0e5110 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/2c739a58bc5e4997aaf9895d2e0e5110 2024-11-10T03:17:19,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742087_1263 (size=12151) 2024-11-10T03:17:19,290 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/A/f5e46cc015074f98b9a17c877e9662c7 2024-11-10T03:17:19,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742088_1264 (size=12207) 2024-11-10T03:17:19,295 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a93817da82260912633bd4e3d359d5b7/A of a93817da82260912633bd4e3d359d5b7 into 2c739a58bc5e4997aaf9895d2e0e5110(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:17:19,296 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a93817da82260912633bd4e3d359d5b7: 2024-11-10T03:17:19,296 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., storeName=a93817da82260912633bd4e3d359d5b7/A, priority=13, startTime=1731208639179; duration=0sec 2024-11-10T03:17:19,296 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T03:17:19,297 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a93817da82260912633bd4e3d359d5b7:A 2024-11-10T03:17:19,297 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T03:17:19,301 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T03:17:19,301 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1541): a93817da82260912633bd4e3d359d5b7/C is initiating minor compaction (all files) 2024-11-10T03:17:19,301 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a93817da82260912633bd4e3d359d5b7/C in TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:19,301 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/20ff6cff7e744e5d8e83553111e4005c, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/cd5d734f48fa4d4aad7563ef742b0d43, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/2adfe9d89dfb4b02ad904ef6c0c3105a] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp, totalSize=35.3 K 2024-11-10T03:17:19,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/B/e38209c6f193440f80dfd2d897ed9059 is 50, key is test_row_0/B:col10/1731208638414/Put/seqid=0 2024-11-10T03:17:19,304 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 20ff6cff7e744e5d8e83553111e4005c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731208636434 2024-11-10T03:17:19,306 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting cd5d734f48fa4d4aad7563ef742b0d43, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731208636552 2024-11-10T03:17:19,306 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2adfe9d89dfb4b02ad904ef6c0c3105a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1731208637140 2024-11-10T03:17:19,310 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/B/161433a883984db6964b963faaaa5197 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/161433a883984db6964b963faaaa5197 2024-11-10T03:17:19,318 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a93817da82260912633bd4e3d359d5b7/B of a93817da82260912633bd4e3d359d5b7 into 161433a883984db6964b963faaaa5197(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:17:19,318 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a93817da82260912633bd4e3d359d5b7: 2024-11-10T03:17:19,318 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., storeName=a93817da82260912633bd4e3d359d5b7/B, priority=13, startTime=1731208639179; duration=0sec 2024-11-10T03:17:19,318 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:17:19,318 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a93817da82260912633bd4e3d359d5b7:B 2024-11-10T03:17:19,326 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a93817da82260912633bd4e3d359d5b7#C#compaction#217 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:17:19,327 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/C/1a1a95d7de704a0f9e2f73d9fd803ff3 is 50, key is test_row_0/C:col10/1731208637140/Put/seqid=0 2024-11-10T03:17:19,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742089_1265 (size=12151) 2024-11-10T03:17:19,341 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/B/e38209c6f193440f80dfd2d897ed9059 2024-11-10T03:17:19,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/C/209b1080fa7b4727bf9a52265ba790e0 is 50, key is test_row_0/C:col10/1731208638414/Put/seqid=0 2024-11-10T03:17:19,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742090_1266 (size=12207) 2024-11-10T03:17:19,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742091_1267 (size=12151) 2024-11-10T03:17:19,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-10T03:17:19,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on a93817da82260912633bd4e3d359d5b7 2024-11-10T03:17:19,576 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. as already flushing 2024-11-10T03:17:19,623 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:19,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49936 deadline: 1731208699618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:19,623 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:19,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49990 deadline: 1731208699619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:19,623 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:19,623 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:19,623 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:19,624 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:19,624 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:19,624 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:19,737 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:19,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49936 deadline: 1731208699733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:19,737 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:19,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49990 deadline: 1731208699733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:19,737 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:19,737 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:19,737 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:19,738 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:19,738 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:19,738 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:19,771 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/C/209b1080fa7b4727bf9a52265ba790e0 2024-11-10T03:17:19,771 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/C/1a1a95d7de704a0f9e2f73d9fd803ff3 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/1a1a95d7de704a0f9e2f73d9fd803ff3 2024-11-10T03:17:19,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/A/f5e46cc015074f98b9a17c877e9662c7 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/f5e46cc015074f98b9a17c877e9662c7 2024-11-10T03:17:19,777 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a93817da82260912633bd4e3d359d5b7/C of a93817da82260912633bd4e3d359d5b7 into 1a1a95d7de704a0f9e2f73d9fd803ff3(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:17:19,777 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a93817da82260912633bd4e3d359d5b7: 2024-11-10T03:17:19,777 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., storeName=a93817da82260912633bd4e3d359d5b7/C, priority=13, startTime=1731208639179; duration=0sec 2024-11-10T03:17:19,777 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:17:19,777 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a93817da82260912633bd4e3d359d5b7:C 2024-11-10T03:17:19,781 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/f5e46cc015074f98b9a17c877e9662c7, entries=150, sequenceid=145, filesize=11.9 K 2024-11-10T03:17:19,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/B/e38209c6f193440f80dfd2d897ed9059 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/e38209c6f193440f80dfd2d897ed9059 2024-11-10T03:17:19,787 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/e38209c6f193440f80dfd2d897ed9059, entries=150, sequenceid=145, filesize=11.9 K 2024-11-10T03:17:19,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/C/209b1080fa7b4727bf9a52265ba790e0 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/209b1080fa7b4727bf9a52265ba790e0 2024-11-10T03:17:19,792 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/209b1080fa7b4727bf9a52265ba790e0, entries=150, sequenceid=145, filesize=11.9 K 2024-11-10T03:17:19,793 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for a93817da82260912633bd4e3d359d5b7 in 582ms, sequenceid=145, compaction requested=false 2024-11-10T03:17:19,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for a93817da82260912633bd4e3d359d5b7: 2024-11-10T03:17:19,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:19,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-11-10T03:17:19,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-11-10T03:17:19,798 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-10T03:17:19,798 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5060 sec 2024-11-10T03:17:19,800 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 1.5150 sec 2024-11-10T03:17:19,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on a93817da82260912633bd4e3d359d5b7 2024-11-10T03:17:19,945 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a93817da82260912633bd4e3d359d5b7 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-10T03:17:19,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a93817da82260912633bd4e3d359d5b7, store=A 2024-11-10T03:17:19,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:17:19,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a93817da82260912633bd4e3d359d5b7, store=B 2024-11-10T03:17:19,947 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:17:19,947 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a93817da82260912633bd4e3d359d5b7, store=C 2024-11-10T03:17:19,947 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:17:19,961 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/A/3c8484b8dab04c36b7a8b2428ee590f5 is 50, key is test_row_0/A:col10/1731208639944/Put/seqid=0 2024-11-10T03:17:19,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742092_1268 (size=16931) 2024-11-10T03:17:19,982 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/A/3c8484b8dab04c36b7a8b2428ee590f5 2024-11-10T03:17:19,995 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/B/424019aec8534f47a56b992fc218405c is 50, key is test_row_0/B:col10/1731208639944/Put/seqid=0 2024-11-10T03:17:20,027 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731208639988, max timestamp=1731208640024Num uniques 859; 2024-11-10T03:17:20,027 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T03:17:20,027 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731208639988, max timestamp=1731208640024Num uniques 859; 2024-11-10T03:17:20,027 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T03:17:20,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742093_1269 (size=12151) 2024-11-10T03:17:20,030 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731208639988, max timestamp=1731208640024Num uniques 859; 2024-11-10T03:17:20,030 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T03:17:20,031 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/B/424019aec8534f47a56b992fc218405c 2024-11-10T03:17:20,040 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/C/7c8240e4279640d1a979a9d8cc4a3ff3 is 50, key is test_row_0/C:col10/1731208639944/Put/seqid=0 2024-11-10T03:17:20,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742094_1270 (size=12151) 2024-11-10T03:17:20,055 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/C/7c8240e4279640d1a979a9d8cc4a3ff3 2024-11-10T03:17:20,061 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/A/3c8484b8dab04c36b7a8b2428ee590f5 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/3c8484b8dab04c36b7a8b2428ee590f5 2024-11-10T03:17:20,066 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/3c8484b8dab04c36b7a8b2428ee590f5, entries=250, sequenceid=160, filesize=16.5 K 2024-11-10T03:17:20,067 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/B/424019aec8534f47a56b992fc218405c as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/424019aec8534f47a56b992fc218405c 2024-11-10T03:17:20,075 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/424019aec8534f47a56b992fc218405c, entries=150, sequenceid=160, filesize=11.9 K 2024-11-10T03:17:20,077 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/C/7c8240e4279640d1a979a9d8cc4a3ff3 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/7c8240e4279640d1a979a9d8cc4a3ff3 2024-11-10T03:17:20,081 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/7c8240e4279640d1a979a9d8cc4a3ff3, entries=150, sequenceid=160, filesize=11.9 K 2024-11-10T03:17:20,082 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for a93817da82260912633bd4e3d359d5b7 in 137ms, sequenceid=160, compaction requested=true 2024-11-10T03:17:20,082 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a93817da82260912633bd4e3d359d5b7: 2024-11-10T03:17:20,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a93817da82260912633bd4e3d359d5b7:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T03:17:20,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:17:20,082 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T03:17:20,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a93817da82260912633bd4e3d359d5b7:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T03:17:20,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:17:20,082 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T03:17:20,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a93817da82260912633bd4e3d359d5b7:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T03:17:20,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T03:17:20,083 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41289 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T03:17:20,083 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36509 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T03:17:20,083 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1541): a93817da82260912633bd4e3d359d5b7/B is initiating minor compaction (all files) 2024-11-10T03:17:20,083 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1541): a93817da82260912633bd4e3d359d5b7/A is initiating minor compaction (all files) 2024-11-10T03:17:20,083 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a93817da82260912633bd4e3d359d5b7/A in TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:20,083 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a93817da82260912633bd4e3d359d5b7/B in TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:20,083 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/2c739a58bc5e4997aaf9895d2e0e5110, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/f5e46cc015074f98b9a17c877e9662c7, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/3c8484b8dab04c36b7a8b2428ee590f5] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp, totalSize=40.3 K 2024-11-10T03:17:20,083 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/161433a883984db6964b963faaaa5197, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/e38209c6f193440f80dfd2d897ed9059, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/424019aec8534f47a56b992fc218405c] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp, totalSize=35.7 K 2024-11-10T03:17:20,084 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 161433a883984db6964b963faaaa5197, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1731208637140 2024-11-10T03:17:20,084 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2c739a58bc5e4997aaf9895d2e0e5110, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1731208637140 2024-11-10T03:17:20,084 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting e38209c6f193440f80dfd2d897ed9059, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731208638414 2024-11-10T03:17:20,084 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting f5e46cc015074f98b9a17c877e9662c7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731208638414 2024-11-10T03:17:20,085 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 424019aec8534f47a56b992fc218405c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1731208639596 2024-11-10T03:17:20,085 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c8484b8dab04c36b7a8b2428ee590f5, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1731208639588 2024-11-10T03:17:20,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on a93817da82260912633bd4e3d359d5b7 2024-11-10T03:17:20,086 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a93817da82260912633bd4e3d359d5b7 3/3 column families, dataSize=120.76 KB heapSize=340.45 KB 2024-11-10T03:17:20,087 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a93817da82260912633bd4e3d359d5b7, store=A 2024-11-10T03:17:20,087 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T03:17:20,087 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a93817da82260912633bd4e3d359d5b7, store=B 2024-11-10T03:17:20,087 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T03:17:20,087 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a93817da82260912633bd4e3d359d5b7, store=C 2024-11-10T03:17:20,087 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T03:17:20,102 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a93817da82260912633bd4e3d359d5b7#B#compaction#222 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:17:20,102 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/B/f72f2a6088c348bc86f9d3d82a314f65 is 50, key is test_row_0/B:col10/1731208639944/Put/seqid=0 2024-11-10T03:17:20,106 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a93817da82260912633bd4e3d359d5b7#A#compaction#223 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:17:20,106 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/A/2ccbbf3ae6e04d1994ae03aad09187c5 is 50, key is test_row_0/A:col10/1731208639944/Put/seqid=0 2024-11-10T03:17:20,115 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/A/888e5822d88043d1ab27355a7806e830 is 50, key is test_row_0/A:col10/1731208640085/Put/seqid=0 2024-11-10T03:17:20,123 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:20,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49936 deadline: 1731208700117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:20,124 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:20,124 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:20,124 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:20,125 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:20,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49990 deadline: 1731208700120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:20,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:20,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:20,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:20,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742095_1271 (size=12459) 2024-11-10T03:17:20,131 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/B/f72f2a6088c348bc86f9d3d82a314f65 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/f72f2a6088c348bc86f9d3d82a314f65 2024-11-10T03:17:20,136 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a93817da82260912633bd4e3d359d5b7/B of a93817da82260912633bd4e3d359d5b7 into f72f2a6088c348bc86f9d3d82a314f65(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:17:20,136 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a93817da82260912633bd4e3d359d5b7: 2024-11-10T03:17:20,136 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., storeName=a93817da82260912633bd4e3d359d5b7/B, priority=13, startTime=1731208640082; duration=0sec 2024-11-10T03:17:20,136 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T03:17:20,136 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a93817da82260912633bd4e3d359d5b7:B 2024-11-10T03:17:20,136 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T03:17:20,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742096_1272 (size=12459) 2024-11-10T03:17:20,138 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36509 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T03:17:20,138 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1541): a93817da82260912633bd4e3d359d5b7/C is initiating minor compaction (all files) 2024-11-10T03:17:20,138 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a93817da82260912633bd4e3d359d5b7/C in TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:20,138 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/1a1a95d7de704a0f9e2f73d9fd803ff3, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/209b1080fa7b4727bf9a52265ba790e0, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/7c8240e4279640d1a979a9d8cc4a3ff3] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp, totalSize=35.7 K 2024-11-10T03:17:20,139 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a1a95d7de704a0f9e2f73d9fd803ff3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1731208637140 2024-11-10T03:17:20,139 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 209b1080fa7b4727bf9a52265ba790e0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731208638414 2024-11-10T03:17:20,139 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c8240e4279640d1a979a9d8cc4a3ff3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1731208639596 2024-11-10T03:17:20,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742097_1273 (size=14541) 2024-11-10T03:17:20,158 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a93817da82260912633bd4e3d359d5b7#C#compaction#225 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:17:20,159 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/C/12792fdd471d400686a67d9b71b85a60 is 50, key is test_row_0/C:col10/1731208639944/Put/seqid=0 2024-11-10T03:17:20,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742098_1274 (size=12459) 2024-11-10T03:17:20,173 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/C/12792fdd471d400686a67d9b71b85a60 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/12792fdd471d400686a67d9b71b85a60 2024-11-10T03:17:20,180 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a93817da82260912633bd4e3d359d5b7/C of a93817da82260912633bd4e3d359d5b7 into 12792fdd471d400686a67d9b71b85a60(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:17:20,180 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a93817da82260912633bd4e3d359d5b7: 2024-11-10T03:17:20,180 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., storeName=a93817da82260912633bd4e3d359d5b7/C, priority=13, startTime=1731208640082; duration=0sec 2024-11-10T03:17:20,180 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:17:20,180 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a93817da82260912633bd4e3d359d5b7:C 2024-11-10T03:17:20,236 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:20,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49936 deadline: 1731208700233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:20,236 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:20,236 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:20,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49990 deadline: 1731208700233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:20,236 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:20,236 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:20,236 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:20,236 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:20,236 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:20,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-10T03:17:20,433 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T03:17:20,433 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T03:17:20,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-10T03:17:20,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-10T03:17:20,435 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T03:17:20,436 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T03:17:20,436 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T03:17:20,446 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:20,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49936 deadline: 1731208700443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:20,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:20,447 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:20,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49990 deadline: 1731208700443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:20,447 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:20,447 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:20,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:20,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:20,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:20,542 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/A/2ccbbf3ae6e04d1994ae03aad09187c5 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/2ccbbf3ae6e04d1994ae03aad09187c5 2024-11-10T03:17:20,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-10T03:17:20,552 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a93817da82260912633bd4e3d359d5b7/A of a93817da82260912633bd4e3d359d5b7 into 2ccbbf3ae6e04d1994ae03aad09187c5(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:17:20,552 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a93817da82260912633bd4e3d359d5b7: 2024-11-10T03:17:20,552 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., storeName=a93817da82260912633bd4e3d359d5b7/A, priority=13, startTime=1731208640082; duration=0sec 2024-11-10T03:17:20,552 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:17:20,552 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a93817da82260912633bd4e3d359d5b7:A 2024-11-10T03:17:20,558 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/A/888e5822d88043d1ab27355a7806e830 2024-11-10T03:17:20,565 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/B/9e1a0ad7592e485a800e6b8ceb55e570 is 50, key is test_row_0/B:col10/1731208640085/Put/seqid=0 2024-11-10T03:17:20,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742099_1275 (size=12151) 2024-11-10T03:17:20,571 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/B/9e1a0ad7592e485a800e6b8ceb55e570 2024-11-10T03:17:20,579 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/C/fbc8ecf0bbbc42c7a8ec5472e99397b8 is 50, key is test_row_0/C:col10/1731208640085/Put/seqid=0 2024-11-10T03:17:20,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742100_1276 (size=12151) 2024-11-10T03:17:20,586 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/C/fbc8ecf0bbbc42c7a8ec5472e99397b8 2024-11-10T03:17:20,588 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-10T03:17:20,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:20,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. as already flushing 2024-11-10T03:17:20,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:20,588 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:20,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:20,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:20,595 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/A/888e5822d88043d1ab27355a7806e830 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/888e5822d88043d1ab27355a7806e830 2024-11-10T03:17:20,599 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/888e5822d88043d1ab27355a7806e830, entries=200, sequenceid=194, filesize=14.2 K 2024-11-10T03:17:20,600 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/B/9e1a0ad7592e485a800e6b8ceb55e570 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/9e1a0ad7592e485a800e6b8ceb55e570 2024-11-10T03:17:20,603 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/9e1a0ad7592e485a800e6b8ceb55e570, entries=150, sequenceid=194, filesize=11.9 K 2024-11-10T03:17:20,604 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/C/fbc8ecf0bbbc42c7a8ec5472e99397b8 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/fbc8ecf0bbbc42c7a8ec5472e99397b8 2024-11-10T03:17:20,607 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/fbc8ecf0bbbc42c7a8ec5472e99397b8, entries=150, sequenceid=194, filesize=11.9 K 2024-11-10T03:17:20,608 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~340.41 KB/348576, currentSize=60.38 KB/61830 for a93817da82260912633bd4e3d359d5b7 in 522ms, sequenceid=194, compaction requested=false 2024-11-10T03:17:20,608 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a93817da82260912633bd4e3d359d5b7: 2024-11-10T03:17:20,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on a93817da82260912633bd4e3d359d5b7 2024-11-10T03:17:20,724 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a93817da82260912633bd4e3d359d5b7 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-10T03:17:20,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a93817da82260912633bd4e3d359d5b7, store=A 2024-11-10T03:17:20,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:17:20,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a93817da82260912633bd4e3d359d5b7, store=B 2024-11-10T03:17:20,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:17:20,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a93817da82260912633bd4e3d359d5b7, store=C 2024-11-10T03:17:20,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:17:20,729 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/A/7c745e31543e49af974aaabb46ec6386 is 50, key is test_row_0/A:col10/1731208640723/Put/seqid=0 2024-11-10T03:17:20,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742101_1277 (size=14541) 2024-11-10T03:17:20,738 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/A/7c745e31543e49af974aaabb46ec6386 2024-11-10T03:17:20,741 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-10T03:17:20,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:20,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. as already flushing 2024-11-10T03:17:20,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:20,741 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:20,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:20,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:20,751 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/B/0a5687c87bbc43f593c0cd808c353856 is 50, key is test_row_0/B:col10/1731208640723/Put/seqid=0 2024-11-10T03:17:20,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-10T03:17:20,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742102_1278 (size=12151) 2024-11-10T03:17:20,783 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:20,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49966 deadline: 1731208700778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:20,783 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:20,783 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:20,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49990 deadline: 1731208700778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:20,783 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:20,783 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:20,783 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:20,783 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:20,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49936 deadline: 1731208700778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:20,783 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:20,783 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:20,784 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:20,784 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:20,784 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:20,785 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:20,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49950 deadline: 1731208700782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:20,786 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:20,786 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:20,786 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:20,788 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:20,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50002 deadline: 1731208700783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:20,788 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:20,788 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:20,788 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:20,895 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:20,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49990 deadline: 1731208700893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:20,895 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:20,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49966 deadline: 1731208700893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:20,895 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:20,896 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:20,896 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:20,896 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:20,896 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:20,896 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:20,896 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:20,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49950 deadline: 1731208700893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:20,896 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:20,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49936 deadline: 1731208700893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:20,896 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-10T03:17:20,896 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:20,896 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:20,896 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:20,896 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:20,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:20,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50002 deadline: 1731208700893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:20,897 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:20,897 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:20,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. as already flushing 2024-11-10T03:17:20,897 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:20,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:20,897 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:20,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:20,897 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:20,897 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:20,897 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:20,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:21,049 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-10T03:17:21,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:21,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. as already flushing 2024-11-10T03:17:21,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:21,049 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:21,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:21,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:21,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-10T03:17:21,105 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:21,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49990 deadline: 1731208701103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:21,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:21,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:21,106 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:21,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:21,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49936 deadline: 1731208701103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:21,106 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:21,106 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:21,106 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:21,106 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:21,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49966 deadline: 1731208701103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:21,106 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:21,106 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:21,106 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:21,106 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:21,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49950 deadline: 1731208701103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:21,107 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:21,107 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:21,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50002 deadline: 1731208701103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:21,107 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:21,107 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:21,107 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:21,107 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:21,107 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:21,171 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/B/0a5687c87bbc43f593c0cd808c353856 2024-11-10T03:17:21,177 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/C/13814e730b8443e09ddd1946a55406df is 50, key is test_row_0/C:col10/1731208640723/Put/seqid=0 2024-11-10T03:17:21,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742103_1279 (size=12151) 2024-11-10T03:17:21,202 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-10T03:17:21,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:21,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. as already flushing 2024-11-10T03:17:21,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:21,202 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:21,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:21,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:21,356 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-10T03:17:21,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:21,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. as already flushing 2024-11-10T03:17:21,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:21,357 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:21,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:21,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:21,414 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:21,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49990 deadline: 1731208701413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:21,415 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:21,415 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:21,415 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:21,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:21,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49950 deadline: 1731208701414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:21,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:21,418 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:21,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50002 deadline: 1731208701414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:21,418 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:21,418 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:21,418 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:21,418 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:21,418 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:21,418 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:21,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:21,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49966 deadline: 1731208701414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:21,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49936 deadline: 1731208701414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:21,419 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:21,419 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:21,419 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:21,420 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:21,420 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:21,420 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:21,510 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-10T03:17:21,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:21,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. as already flushing 2024-11-10T03:17:21,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:21,510 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:21,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:21,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:21,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-10T03:17:21,586 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/C/13814e730b8443e09ddd1946a55406df 2024-11-10T03:17:21,593 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/A/7c745e31543e49af974aaabb46ec6386 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/7c745e31543e49af974aaabb46ec6386 2024-11-10T03:17:21,597 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/7c745e31543e49af974aaabb46ec6386, entries=200, sequenceid=210, filesize=14.2 K 2024-11-10T03:17:21,598 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/B/0a5687c87bbc43f593c0cd808c353856 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/0a5687c87bbc43f593c0cd808c353856 2024-11-10T03:17:21,603 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/0a5687c87bbc43f593c0cd808c353856, entries=150, sequenceid=210, filesize=11.9 K 2024-11-10T03:17:21,604 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/C/13814e730b8443e09ddd1946a55406df as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/13814e730b8443e09ddd1946a55406df 2024-11-10T03:17:21,613 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/13814e730b8443e09ddd1946a55406df, entries=150, sequenceid=210, filesize=11.9 K 2024-11-10T03:17:21,615 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for a93817da82260912633bd4e3d359d5b7 in 890ms, sequenceid=210, compaction requested=true 2024-11-10T03:17:21,615 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a93817da82260912633bd4e3d359d5b7: 2024-11-10T03:17:21,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a93817da82260912633bd4e3d359d5b7:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T03:17:21,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:17:21,615 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T03:17:21,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a93817da82260912633bd4e3d359d5b7:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T03:17:21,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:17:21,615 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T03:17:21,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a93817da82260912633bd4e3d359d5b7:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T03:17:21,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T03:17:21,617 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T03:17:21,617 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1541): a93817da82260912633bd4e3d359d5b7/B is initiating minor compaction (all files) 2024-11-10T03:17:21,617 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a93817da82260912633bd4e3d359d5b7/B in TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:21,617 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/f72f2a6088c348bc86f9d3d82a314f65, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/9e1a0ad7592e485a800e6b8ceb55e570, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/0a5687c87bbc43f593c0cd808c353856] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp, totalSize=35.9 K 2024-11-10T03:17:21,617 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41541 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T03:17:21,617 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1541): a93817da82260912633bd4e3d359d5b7/A is initiating minor compaction (all files) 2024-11-10T03:17:21,617 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a93817da82260912633bd4e3d359d5b7/A in TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:21,617 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/2ccbbf3ae6e04d1994ae03aad09187c5, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/888e5822d88043d1ab27355a7806e830, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/7c745e31543e49af974aaabb46ec6386] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp, totalSize=40.6 K 2024-11-10T03:17:21,617 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting f72f2a6088c348bc86f9d3d82a314f65, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1731208639596 2024-11-10T03:17:21,618 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2ccbbf3ae6e04d1994ae03aad09187c5, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1731208639596 2024-11-10T03:17:21,618 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e1a0ad7592e485a800e6b8ceb55e570, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731208640075 2024-11-10T03:17:21,618 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a5687c87bbc43f593c0cd808c353856, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731208640116 2024-11-10T03:17:21,618 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 888e5822d88043d1ab27355a7806e830, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731208640066 2024-11-10T03:17:21,618 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c745e31543e49af974aaabb46ec6386, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731208640109 2024-11-10T03:17:21,625 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a93817da82260912633bd4e3d359d5b7#A#compaction#231 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:17:21,625 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a93817da82260912633bd4e3d359d5b7#B#compaction#232 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:17:21,626 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/A/48f574eceb9b4d3d8e735cb3d27bba2e is 50, key is test_row_0/A:col10/1731208640723/Put/seqid=0 2024-11-10T03:17:21,626 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/B/6fa29ec7b85243fabb074283dc9055da is 50, key is test_row_0/B:col10/1731208640723/Put/seqid=0 2024-11-10T03:17:21,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742105_1281 (size=12561) 2024-11-10T03:17:21,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742104_1280 (size=12561) 2024-11-10T03:17:21,637 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/B/6fa29ec7b85243fabb074283dc9055da as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/6fa29ec7b85243fabb074283dc9055da 2024-11-10T03:17:21,641 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a93817da82260912633bd4e3d359d5b7/B of a93817da82260912633bd4e3d359d5b7 into 6fa29ec7b85243fabb074283dc9055da(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:17:21,641 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a93817da82260912633bd4e3d359d5b7: 2024-11-10T03:17:21,642 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., storeName=a93817da82260912633bd4e3d359d5b7/B, priority=13, startTime=1731208641615; duration=0sec 2024-11-10T03:17:21,642 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T03:17:21,642 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a93817da82260912633bd4e3d359d5b7:B 2024-11-10T03:17:21,642 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T03:17:21,645 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T03:17:21,645 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1541): a93817da82260912633bd4e3d359d5b7/C is initiating minor compaction (all files) 2024-11-10T03:17:21,645 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a93817da82260912633bd4e3d359d5b7/C in TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:21,645 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/12792fdd471d400686a67d9b71b85a60, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/fbc8ecf0bbbc42c7a8ec5472e99397b8, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/13814e730b8443e09ddd1946a55406df] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp, totalSize=35.9 K 2024-11-10T03:17:21,646 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 12792fdd471d400686a67d9b71b85a60, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1731208639596 2024-11-10T03:17:21,646 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting fbc8ecf0bbbc42c7a8ec5472e99397b8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731208640075 2024-11-10T03:17:21,646 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 13814e730b8443e09ddd1946a55406df, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731208640116 2024-11-10T03:17:21,654 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a93817da82260912633bd4e3d359d5b7#C#compaction#233 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:17:21,654 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/C/83d8b1ff5a2a4a49b0c3a9201f7ea446 is 50, key is test_row_0/C:col10/1731208640723/Put/seqid=0 2024-11-10T03:17:21,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742106_1282 (size=12561) 2024-11-10T03:17:21,664 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-10T03:17:21,664 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/C/83d8b1ff5a2a4a49b0c3a9201f7ea446 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/83d8b1ff5a2a4a49b0c3a9201f7ea446 2024-11-10T03:17:21,664 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-10T03:17:21,664 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-10T03:17:21,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:21,664 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing a93817da82260912633bd4e3d359d5b7 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-10T03:17:21,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a93817da82260912633bd4e3d359d5b7, store=A 2024-11-10T03:17:21,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:17:21,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a93817da82260912633bd4e3d359d5b7, store=B 2024-11-10T03:17:21,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:17:21,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a93817da82260912633bd4e3d359d5b7, store=C 2024-11-10T03:17:21,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:17:21,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/A/37a0d8e72a3c4ab1a1bde45715c1233b is 50, key is test_row_0/A:col10/1731208640782/Put/seqid=0 2024-11-10T03:17:21,673 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a93817da82260912633bd4e3d359d5b7/C of a93817da82260912633bd4e3d359d5b7 into 83d8b1ff5a2a4a49b0c3a9201f7ea446(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:17:21,673 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a93817da82260912633bd4e3d359d5b7: 2024-11-10T03:17:21,673 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., storeName=a93817da82260912633bd4e3d359d5b7/C, priority=13, startTime=1731208641615; duration=0sec 2024-11-10T03:17:21,673 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:17:21,673 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a93817da82260912633bd4e3d359d5b7:C 2024-11-10T03:17:21,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742107_1283 (size=12151) 2024-11-10T03:17:21,934 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. as already flushing 2024-11-10T03:17:21,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on a93817da82260912633bd4e3d359d5b7 2024-11-10T03:17:21,961 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:21,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49950 deadline: 1731208701953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:21,961 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:21,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49990 deadline: 1731208701953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:21,962 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:21,962 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:21,962 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:21,963 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:21,963 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:21,963 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:21,963 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:21,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50002 deadline: 1731208701954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:21,963 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:21,963 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:21,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49936 deadline: 1731208701956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:21,963 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:21,963 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:21,964 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:21,964 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:21,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:21,964 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:21,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49966 deadline: 1731208701957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:21,964 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:21,964 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:21,964 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:22,050 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/A/48f574eceb9b4d3d8e735cb3d27bba2e as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/48f574eceb9b4d3d8e735cb3d27bba2e 2024-11-10T03:17:22,058 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a93817da82260912633bd4e3d359d5b7/A of a93817da82260912633bd4e3d359d5b7 into 48f574eceb9b4d3d8e735cb3d27bba2e(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:17:22,059 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a93817da82260912633bd4e3d359d5b7: 2024-11-10T03:17:22,059 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., storeName=a93817da82260912633bd4e3d359d5b7/A, priority=13, startTime=1731208641615; duration=0sec 2024-11-10T03:17:22,059 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:17:22,059 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a93817da82260912633bd4e3d359d5b7:A 2024-11-10T03:17:22,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:22,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49950 deadline: 1731208702074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:22,078 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:22,078 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:22,078 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:22,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:22,079 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/A/37a0d8e72a3c4ab1a1bde45715c1233b 2024-11-10T03:17:22,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50002 deadline: 1731208702074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:22,080 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:22,080 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:22,080 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:22,081 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:22,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49966 deadline: 1731208702074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:22,082 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:22,082 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:22,082 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:22,082 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:22,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49936 deadline: 1731208702075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:22,082 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:22,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49990 deadline: 1731208702075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:22,083 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:22,083 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:22,083 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:22,083 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:22,083 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:22,083 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:22,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/B/3112fc8fa9d84f21925a2dfe86f62768 is 50, key is test_row_0/B:col10/1731208640782/Put/seqid=0 2024-11-10T03:17:22,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742108_1284 (size=12151) 2024-11-10T03:17:22,147 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/B/3112fc8fa9d84f21925a2dfe86f62768 2024-11-10T03:17:22,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/C/0e1701de081640d08b091a10a42805f8 is 50, key is test_row_0/C:col10/1731208640782/Put/seqid=0 2024-11-10T03:17:22,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742109_1285 (size=12151) 2024-11-10T03:17:22,209 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/C/0e1701de081640d08b091a10a42805f8 2024-11-10T03:17:22,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/A/37a0d8e72a3c4ab1a1bde45715c1233b as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/37a0d8e72a3c4ab1a1bde45715c1233b 2024-11-10T03:17:22,217 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/37a0d8e72a3c4ab1a1bde45715c1233b, entries=150, sequenceid=230, filesize=11.9 K 2024-11-10T03:17:22,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/B/3112fc8fa9d84f21925a2dfe86f62768 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/3112fc8fa9d84f21925a2dfe86f62768 2024-11-10T03:17:22,221 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/3112fc8fa9d84f21925a2dfe86f62768, entries=150, sequenceid=230, filesize=11.9 K 2024-11-10T03:17:22,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/C/0e1701de081640d08b091a10a42805f8 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/0e1701de081640d08b091a10a42805f8 2024-11-10T03:17:22,227 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/0e1701de081640d08b091a10a42805f8, entries=150, sequenceid=230, filesize=11.9 K 2024-11-10T03:17:22,228 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for a93817da82260912633bd4e3d359d5b7 in 564ms, sequenceid=230, compaction requested=false 2024-11-10T03:17:22,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for a93817da82260912633bd4e3d359d5b7: 2024-11-10T03:17:22,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:22,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-10T03:17:22,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-10T03:17:22,232 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-10T03:17:22,232 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7940 sec 2024-11-10T03:17:22,233 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 1.7990 sec 2024-11-10T03:17:22,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on a93817da82260912633bd4e3d359d5b7 2024-11-10T03:17:22,285 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a93817da82260912633bd4e3d359d5b7 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-10T03:17:22,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a93817da82260912633bd4e3d359d5b7, store=A 2024-11-10T03:17:22,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:17:22,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a93817da82260912633bd4e3d359d5b7, store=B 2024-11-10T03:17:22,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:17:22,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a93817da82260912633bd4e3d359d5b7, store=C 2024-11-10T03:17:22,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:17:22,290 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/A/b6181718ddb349ffb4f4c42fe8052acf is 50, key is test_row_0/A:col10/1731208641954/Put/seqid=0 2024-11-10T03:17:22,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742110_1286 (size=14541) 2024-11-10T03:17:22,342 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:22,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49950 deadline: 1731208702336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:22,342 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:22,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50002 deadline: 1731208702336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:22,342 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:22,342 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:22,342 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:22,343 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:22,343 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:22,343 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:22,346 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:22,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49966 deadline: 1731208702341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:22,347 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:22,347 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:22,347 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:22,348 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:22,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49990 deadline: 1731208702342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:22,348 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:22,348 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:22,348 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:22,348 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:22,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49936 deadline: 1731208702343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:22,348 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:22,348 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:22,348 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:22,455 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:22,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49950 deadline: 1731208702453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:22,456 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:22,456 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:22,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50002 deadline: 1731208702453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:22,456 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:22,456 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:22,456 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:22,456 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:22,456 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:22,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49966 deadline: 1731208702453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:22,456 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:22,456 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:22,456 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:22,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49990 deadline: 1731208702453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:22,456 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:22,456 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:22,456 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:22,456 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:22,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:22,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49936 deadline: 1731208702453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:22,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:22,457 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:22,457 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:22,457 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:22,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-10T03:17:22,583 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T03:17:22,584 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T03:17:22,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-10T03:17:22,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-10T03:17:22,590 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T03:17:22,591 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T03:17:22,591 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T03:17:22,667 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:22,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49950 deadline: 1731208702663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:22,667 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:22,667 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:22,667 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:22,667 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:22,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50002 deadline: 1731208702663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:22,668 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:22,668 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:22,668 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:22,668 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:22,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49990 deadline: 1731208702664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:22,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:22,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:22,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:22,668 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:22,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49936 deadline: 1731208702664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:22,669 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:22,669 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:22,669 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:22,669 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:22,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49966 deadline: 1731208702664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:22,669 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:22,669 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:22,669 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:22,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-10T03:17:22,700 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/A/b6181718ddb349ffb4f4c42fe8052acf 2024-11-10T03:17:22,708 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/B/f2cb4b008b8c43b2b4769855b6cd679d is 50, key is test_row_0/B:col10/1731208641954/Put/seqid=0 2024-11-10T03:17:22,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742111_1287 (size=12151) 2024-11-10T03:17:22,746 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-10T03:17:22,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:22,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. as already flushing 2024-11-10T03:17:22,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:22,747 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:22,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:22,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:22,899 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-10T03:17:22,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:22,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. as already flushing 2024-11-10T03:17:22,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:22,900 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:22,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:22,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:22,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-10T03:17:22,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:22,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49950 deadline: 1731208702973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:22,976 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:22,976 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:22,976 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:22,976 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:22,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50002 deadline: 1731208702973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:22,976 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:22,976 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:22,976 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:22,976 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:22,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49990 deadline: 1731208702973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:22,977 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:22,977 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:22,977 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:22,977 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:22,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49936 deadline: 1731208702973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:22,977 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:22,977 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:22,977 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:22,977 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:22,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49966 deadline: 1731208702973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:22,977 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:22,977 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:22,977 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:23,052 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-10T03:17:23,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:23,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. as already flushing 2024-11-10T03:17:23,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:23,052 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:23,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:23,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:23,119 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/B/f2cb4b008b8c43b2b4769855b6cd679d 2024-11-10T03:17:23,125 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/C/87f9a104a38041dc9c1a25c92a18c696 is 50, key is test_row_0/C:col10/1731208641954/Put/seqid=0 2024-11-10T03:17:23,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742112_1288 (size=12151) 2024-11-10T03:17:23,205 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-10T03:17:23,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:23,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. as already flushing 2024-11-10T03:17:23,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:23,205 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:23,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:23,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:23,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-10T03:17:23,360 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-10T03:17:23,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:23,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. as already flushing 2024-11-10T03:17:23,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:23,360 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:23,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:23,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:23,485 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:23,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49950 deadline: 1731208703483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:23,485 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:23,485 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:23,485 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:23,485 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:23,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50002 deadline: 1731208703483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:23,485 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:23,485 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:23,485 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:23,486 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:23,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49990 deadline: 1731208703483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:23,486 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:23,486 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:23,486 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:23,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49936 deadline: 1731208703483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:23,486 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:23,486 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:23,486 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:23,486 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:23,486 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:23,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49966 deadline: 1731208703484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:23,486 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:23,486 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:23,486 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:23,513 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-10T03:17:23,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:23,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. as already flushing 2024-11-10T03:17:23,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:23,513 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:23,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:23,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:23,530 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/C/87f9a104a38041dc9c1a25c92a18c696 2024-11-10T03:17:23,534 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/A/b6181718ddb349ffb4f4c42fe8052acf as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/b6181718ddb349ffb4f4c42fe8052acf 2024-11-10T03:17:23,538 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/b6181718ddb349ffb4f4c42fe8052acf, entries=200, sequenceid=247, filesize=14.2 K 2024-11-10T03:17:23,538 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/B/f2cb4b008b8c43b2b4769855b6cd679d as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/f2cb4b008b8c43b2b4769855b6cd679d 2024-11-10T03:17:23,542 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/f2cb4b008b8c43b2b4769855b6cd679d, entries=150, sequenceid=247, filesize=11.9 K 2024-11-10T03:17:23,543 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/C/87f9a104a38041dc9c1a25c92a18c696 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/87f9a104a38041dc9c1a25c92a18c696 2024-11-10T03:17:23,546 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/87f9a104a38041dc9c1a25c92a18c696, entries=150, sequenceid=247, filesize=11.9 K 2024-11-10T03:17:23,547 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for a93817da82260912633bd4e3d359d5b7 in 1262ms, sequenceid=247, compaction requested=true 2024-11-10T03:17:23,547 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a93817da82260912633bd4e3d359d5b7: 2024-11-10T03:17:23,547 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T03:17:23,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a93817da82260912633bd4e3d359d5b7:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T03:17:23,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:17:23,547 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T03:17:23,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a93817da82260912633bd4e3d359d5b7:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T03:17:23,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:17:23,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a93817da82260912633bd4e3d359d5b7:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T03:17:23,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T03:17:23,548 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T03:17:23,548 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39253 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T03:17:23,548 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1541): a93817da82260912633bd4e3d359d5b7/A is initiating minor compaction (all files) 2024-11-10T03:17:23,548 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1541): a93817da82260912633bd4e3d359d5b7/B is initiating minor compaction (all files) 2024-11-10T03:17:23,548 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a93817da82260912633bd4e3d359d5b7/A in TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:23,548 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/48f574eceb9b4d3d8e735cb3d27bba2e, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/37a0d8e72a3c4ab1a1bde45715c1233b, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/b6181718ddb349ffb4f4c42fe8052acf] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp, totalSize=38.3 K 2024-11-10T03:17:23,548 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a93817da82260912633bd4e3d359d5b7/B in TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:23,549 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/6fa29ec7b85243fabb074283dc9055da, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/3112fc8fa9d84f21925a2dfe86f62768, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/f2cb4b008b8c43b2b4769855b6cd679d] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp, totalSize=36.0 K 2024-11-10T03:17:23,549 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 48f574eceb9b4d3d8e735cb3d27bba2e, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731208640116 2024-11-10T03:17:23,549 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 6fa29ec7b85243fabb074283dc9055da, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731208640116 2024-11-10T03:17:23,549 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 37a0d8e72a3c4ab1a1bde45715c1233b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731208640777 2024-11-10T03:17:23,550 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 3112fc8fa9d84f21925a2dfe86f62768, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731208640777 2024-11-10T03:17:23,550 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting b6181718ddb349ffb4f4c42fe8052acf, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731208641949 2024-11-10T03:17:23,550 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting f2cb4b008b8c43b2b4769855b6cd679d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731208641954 2024-11-10T03:17:23,567 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a93817da82260912633bd4e3d359d5b7#A#compaction#240 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:17:23,567 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/A/4feb3b35091d42c8a86da79b7da7dfe9 is 50, key is test_row_0/A:col10/1731208641954/Put/seqid=0 2024-11-10T03:17:23,572 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a93817da82260912633bd4e3d359d5b7#B#compaction#241 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:17:23,573 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/B/26c6a8433dbc47e3be5abbc61534cbc8 is 50, key is test_row_0/B:col10/1731208641954/Put/seqid=0 2024-11-10T03:17:23,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742113_1289 (size=12663) 2024-11-10T03:17:23,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742114_1290 (size=12663) 2024-11-10T03:17:23,584 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/A/4feb3b35091d42c8a86da79b7da7dfe9 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/4feb3b35091d42c8a86da79b7da7dfe9 2024-11-10T03:17:23,593 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a93817da82260912633bd4e3d359d5b7/A of a93817da82260912633bd4e3d359d5b7 into 4feb3b35091d42c8a86da79b7da7dfe9(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:17:23,593 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a93817da82260912633bd4e3d359d5b7: 2024-11-10T03:17:23,593 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., storeName=a93817da82260912633bd4e3d359d5b7/A, priority=13, startTime=1731208643547; duration=0sec 2024-11-10T03:17:23,594 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T03:17:23,594 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a93817da82260912633bd4e3d359d5b7:A 2024-11-10T03:17:23,594 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T03:17:23,595 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T03:17:23,595 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1541): a93817da82260912633bd4e3d359d5b7/C is initiating minor compaction (all files) 2024-11-10T03:17:23,596 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a93817da82260912633bd4e3d359d5b7/C in TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:23,596 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/83d8b1ff5a2a4a49b0c3a9201f7ea446, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/0e1701de081640d08b091a10a42805f8, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/87f9a104a38041dc9c1a25c92a18c696] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp, totalSize=36.0 K 2024-11-10T03:17:23,596 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 83d8b1ff5a2a4a49b0c3a9201f7ea446, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731208640116 2024-11-10T03:17:23,596 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e1701de081640d08b091a10a42805f8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731208640777 2024-11-10T03:17:23,596 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 87f9a104a38041dc9c1a25c92a18c696, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731208641954 2024-11-10T03:17:23,613 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a93817da82260912633bd4e3d359d5b7#C#compaction#242 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:17:23,613 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/C/8e6259fdb1df4cf99e5fd6801c7b1c4c is 50, key is test_row_0/C:col10/1731208641954/Put/seqid=0 2024-11-10T03:17:23,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742115_1291 (size=12663) 2024-11-10T03:17:23,667 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-10T03:17:23,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:23,667 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing a93817da82260912633bd4e3d359d5b7 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-10T03:17:23,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a93817da82260912633bd4e3d359d5b7, store=A 2024-11-10T03:17:23,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:17:23,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a93817da82260912633bd4e3d359d5b7, store=B 2024-11-10T03:17:23,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:17:23,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a93817da82260912633bd4e3d359d5b7, store=C 2024-11-10T03:17:23,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:17:23,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/A/d4a3ff513c3b4300a5533423600fe6dd is 50, key is test_row_0/A:col10/1731208642341/Put/seqid=0 2024-11-10T03:17:23,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742116_1292 (size=12301) 2024-11-10T03:17:23,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-10T03:17:24,002 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/B/26c6a8433dbc47e3be5abbc61534cbc8 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/26c6a8433dbc47e3be5abbc61534cbc8 2024-11-10T03:17:24,013 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a93817da82260912633bd4e3d359d5b7/B of a93817da82260912633bd4e3d359d5b7 into 26c6a8433dbc47e3be5abbc61534cbc8(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:17:24,013 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a93817da82260912633bd4e3d359d5b7: 2024-11-10T03:17:24,013 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., storeName=a93817da82260912633bd4e3d359d5b7/B, priority=13, startTime=1731208643547; duration=0sec 2024-11-10T03:17:24,013 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:17:24,013 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a93817da82260912633bd4e3d359d5b7:B 2024-11-10T03:17:24,040 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/C/8e6259fdb1df4cf99e5fd6801c7b1c4c as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/8e6259fdb1df4cf99e5fd6801c7b1c4c 2024-11-10T03:17:24,047 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a93817da82260912633bd4e3d359d5b7/C of a93817da82260912633bd4e3d359d5b7 into 8e6259fdb1df4cf99e5fd6801c7b1c4c(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:17:24,047 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a93817da82260912633bd4e3d359d5b7: 2024-11-10T03:17:24,047 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., storeName=a93817da82260912633bd4e3d359d5b7/C, priority=13, startTime=1731208643547; duration=0sec 2024-11-10T03:17:24,047 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:17:24,047 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a93817da82260912633bd4e3d359d5b7:C 2024-11-10T03:17:24,077 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/A/d4a3ff513c3b4300a5533423600fe6dd 2024-11-10T03:17:24,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/B/7b3d6903bfed4fbc8808078c2a2a8b8e is 50, key is test_row_0/B:col10/1731208642341/Put/seqid=0 2024-11-10T03:17:24,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742117_1293 (size=12301) 2024-11-10T03:17:24,129 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/B/7b3d6903bfed4fbc8808078c2a2a8b8e 2024-11-10T03:17:24,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/C/78598c9ae8dd487dacf716cad4a34c56 is 50, key is test_row_0/C:col10/1731208642341/Put/seqid=0 2024-11-10T03:17:24,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742118_1294 (size=12301) 2024-11-10T03:17:24,162 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/C/78598c9ae8dd487dacf716cad4a34c56 2024-11-10T03:17:24,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/A/d4a3ff513c3b4300a5533423600fe6dd as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/d4a3ff513c3b4300a5533423600fe6dd 2024-11-10T03:17:24,173 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/d4a3ff513c3b4300a5533423600fe6dd, entries=150, sequenceid=266, filesize=12.0 K 2024-11-10T03:17:24,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/B/7b3d6903bfed4fbc8808078c2a2a8b8e as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/7b3d6903bfed4fbc8808078c2a2a8b8e 2024-11-10T03:17:24,178 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/7b3d6903bfed4fbc8808078c2a2a8b8e, entries=150, sequenceid=266, filesize=12.0 K 2024-11-10T03:17:24,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/C/78598c9ae8dd487dacf716cad4a34c56 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/78598c9ae8dd487dacf716cad4a34c56 2024-11-10T03:17:24,189 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/78598c9ae8dd487dacf716cad4a34c56, entries=150, sequenceid=266, filesize=12.0 K 2024-11-10T03:17:24,190 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=0 B/0 for a93817da82260912633bd4e3d359d5b7 in 523ms, sequenceid=266, compaction requested=false 2024-11-10T03:17:24,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for a93817da82260912633bd4e3d359d5b7: 2024-11-10T03:17:24,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:24,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-10T03:17:24,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-10T03:17:24,194 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-10T03:17:24,194 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6010 sec 2024-11-10T03:17:24,197 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 1.6110 sec 2024-11-10T03:17:24,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on a93817da82260912633bd4e3d359d5b7 2024-11-10T03:17:24,497 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a93817da82260912633bd4e3d359d5b7 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-10T03:17:24,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a93817da82260912633bd4e3d359d5b7, store=A 2024-11-10T03:17:24,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:17:24,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a93817da82260912633bd4e3d359d5b7, store=B 2024-11-10T03:17:24,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:17:24,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a93817da82260912633bd4e3d359d5b7, store=C 2024-11-10T03:17:24,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:17:24,502 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/A/95ba08c1765e41d08fec4d9cf495bcc2 is 50, key is test_row_0/A:col10/1731208644493/Put/seqid=0 2024-11-10T03:17:24,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742119_1295 (size=12301) 2024-11-10T03:17:24,552 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731208644509, max timestamp=1731208644549Num uniques 859; 2024-11-10T03:17:24,552 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731208644509, max timestamp=1731208644549Num uniques 859; 2024-11-10T03:17:24,552 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-10T03:17:24,552 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-10T03:17:24,554 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731208644547, max timestamp=1731208644549Num uniques 859; 2024-11-10T03:17:24,554 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T03:17:24,574 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:24,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50002 deadline: 1731208704568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:24,574 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:24,574 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:24,574 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:24,574 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:24,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49936 deadline: 1731208704569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:24,575 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:24,575 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:24,575 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:24,576 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:24,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49950 deadline: 1731208704570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:24,576 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:24,577 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:24,577 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:24,580 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:24,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49966 deadline: 1731208704574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:24,580 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:24,580 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:24,580 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:24,580 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:24,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49990 deadline: 1731208704574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:24,581 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:24,581 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:24,581 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:24,687 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:24,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50002 deadline: 1731208704682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:24,687 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:24,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49936 deadline: 1731208704683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:24,687 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:24,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49950 deadline: 1731208704683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:24,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:24,688 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:24,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49966 deadline: 1731208704683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:24,688 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:24,688 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:24,688 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:24,688 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:24,688 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:24,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:24,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49990 deadline: 1731208704683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:24,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:24,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:24,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:24,689 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:24,689 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:24,689 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:24,689 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:24,689 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:24,689 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:24,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-10T03:17:24,733 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T03:17:24,733 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T03:17:24,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-10T03:17:24,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-10T03:17:24,734 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T03:17:24,735 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T03:17:24,735 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T03:17:24,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-10T03:17:24,887 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-10T03:17:24,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:24,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. as already flushing 2024-11-10T03:17:24,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:24,887 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:24,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:24,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:24,898 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:24,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50002 deadline: 1731208704894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:24,898 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:24,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49966 deadline: 1731208704894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:24,898 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:24,898 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:24,898 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:24,898 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:24,898 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:24,898 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:24,898 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:24,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49950 deadline: 1731208704895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:24,899 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:24,899 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:24,899 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:24,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49990 deadline: 1731208704895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:24,899 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:24,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:24,899 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:24,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:24,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:24,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49936 deadline: 1731208704895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:24,899 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:24,899 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:24,899 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:24,915 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/A/95ba08c1765e41d08fec4d9cf495bcc2 2024-11-10T03:17:24,921 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/B/11f812621f454414b63bdffe82535e9d is 50, key is test_row_0/B:col10/1731208644493/Put/seqid=0 2024-11-10T03:17:24,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742120_1296 (size=12301) 2024-11-10T03:17:25,041 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-10T03:17:25,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:25,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. as already flushing 2024-11-10T03:17:25,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:25,041 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:25,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:25,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:25,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-10T03:17:25,194 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-10T03:17:25,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:25,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. as already flushing 2024-11-10T03:17:25,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:25,196 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:25,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:25,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:25,208 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:25,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49950 deadline: 1731208705204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:25,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:25,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49990 deadline: 1731208705204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:25,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:25,210 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:25,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49966 deadline: 1731208705204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:25,210 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:25,210 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:25,210 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:25,210 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:25,210 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:25,212 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:25,212 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:25,212 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:25,212 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:25,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50002 deadline: 1731208705208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:25,216 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:25,216 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:25,216 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:25,218 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:25,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49936 deadline: 1731208705216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:25,219 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:25,219 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:25,219 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:25,324 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/B/11f812621f454414b63bdffe82535e9d 2024-11-10T03:17:25,351 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-10T03:17:25,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:25,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. as already flushing 2024-11-10T03:17:25,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:25,351 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:25,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:25,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:25,358 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/C/4241d7d441324330b20dc0395d97ccbc is 50, key is test_row_0/C:col10/1731208644493/Put/seqid=0 2024-11-10T03:17:25,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742121_1297 (size=12301) 2024-11-10T03:17:25,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-10T03:17:25,363 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/C/4241d7d441324330b20dc0395d97ccbc 2024-11-10T03:17:25,368 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/A/95ba08c1765e41d08fec4d9cf495bcc2 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/95ba08c1765e41d08fec4d9cf495bcc2 2024-11-10T03:17:25,373 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/95ba08c1765e41d08fec4d9cf495bcc2, entries=150, sequenceid=279, filesize=12.0 K 2024-11-10T03:17:25,374 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/B/11f812621f454414b63bdffe82535e9d as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/11f812621f454414b63bdffe82535e9d 2024-11-10T03:17:25,379 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/11f812621f454414b63bdffe82535e9d, entries=150, sequenceid=279, filesize=12.0 K 2024-11-10T03:17:25,381 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/C/4241d7d441324330b20dc0395d97ccbc as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/4241d7d441324330b20dc0395d97ccbc 2024-11-10T03:17:25,384 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/4241d7d441324330b20dc0395d97ccbc, entries=150, sequenceid=279, filesize=12.0 K 2024-11-10T03:17:25,385 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=131.94 KB/135110 for a93817da82260912633bd4e3d359d5b7 in 888ms, sequenceid=279, compaction requested=true 2024-11-10T03:17:25,385 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a93817da82260912633bd4e3d359d5b7: 2024-11-10T03:17:25,385 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T03:17:25,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a93817da82260912633bd4e3d359d5b7:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T03:17:25,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:17:25,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a93817da82260912633bd4e3d359d5b7:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T03:17:25,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T03:17:25,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a93817da82260912633bd4e3d359d5b7:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T03:17:25,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-10T03:17:25,386 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T03:17:25,387 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37265 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T03:17:25,387 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1541): a93817da82260912633bd4e3d359d5b7/A is initiating minor compaction (all files) 2024-11-10T03:17:25,387 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a93817da82260912633bd4e3d359d5b7/A in TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:25,387 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/4feb3b35091d42c8a86da79b7da7dfe9, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/d4a3ff513c3b4300a5533423600fe6dd, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/95ba08c1765e41d08fec4d9cf495bcc2] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp, totalSize=36.4 K 2024-11-10T03:17:25,387 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37265 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T03:17:25,387 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1541): a93817da82260912633bd4e3d359d5b7/B is initiating minor compaction (all files) 2024-11-10T03:17:25,387 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a93817da82260912633bd4e3d359d5b7/B in TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:25,388 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/26c6a8433dbc47e3be5abbc61534cbc8, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/7b3d6903bfed4fbc8808078c2a2a8b8e, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/11f812621f454414b63bdffe82535e9d] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp, totalSize=36.4 K 2024-11-10T03:17:25,388 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4feb3b35091d42c8a86da79b7da7dfe9, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731208641954 2024-11-10T03:17:25,389 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 26c6a8433dbc47e3be5abbc61534cbc8, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731208641954 2024-11-10T03:17:25,389 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting d4a3ff513c3b4300a5533423600fe6dd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1731208642340 2024-11-10T03:17:25,389 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b3d6903bfed4fbc8808078c2a2a8b8e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1731208642340 2024-11-10T03:17:25,389 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 95ba08c1765e41d08fec4d9cf495bcc2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1731208644493 2024-11-10T03:17:25,389 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 11f812621f454414b63bdffe82535e9d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1731208644493 2024-11-10T03:17:25,401 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a93817da82260912633bd4e3d359d5b7#A#compaction#249 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:17:25,401 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/A/58a2785993274f8295c8e084d40e97eb is 50, key is test_row_0/A:col10/1731208644493/Put/seqid=0 2024-11-10T03:17:25,403 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a93817da82260912633bd4e3d359d5b7#B#compaction#250 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:17:25,405 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/B/8f0f041344c34abeb0cbaa906ef503af is 50, key is test_row_0/B:col10/1731208644493/Put/seqid=0 2024-11-10T03:17:25,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742123_1299 (size=12915) 2024-11-10T03:17:25,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742122_1298 (size=12915) 2024-11-10T03:17:25,428 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/A/58a2785993274f8295c8e084d40e97eb as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/58a2785993274f8295c8e084d40e97eb 2024-11-10T03:17:25,434 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a93817da82260912633bd4e3d359d5b7/A of a93817da82260912633bd4e3d359d5b7 into 58a2785993274f8295c8e084d40e97eb(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:17:25,434 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a93817da82260912633bd4e3d359d5b7: 2024-11-10T03:17:25,434 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., storeName=a93817da82260912633bd4e3d359d5b7/A, priority=13, startTime=1731208645385; duration=0sec 2024-11-10T03:17:25,434 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T03:17:25,434 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a93817da82260912633bd4e3d359d5b7:A 2024-11-10T03:17:25,434 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T03:17:25,435 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37265 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T03:17:25,435 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1541): a93817da82260912633bd4e3d359d5b7/C is initiating minor compaction (all files) 2024-11-10T03:17:25,436 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a93817da82260912633bd4e3d359d5b7/C in TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:25,436 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/8e6259fdb1df4cf99e5fd6801c7b1c4c, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/78598c9ae8dd487dacf716cad4a34c56, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/4241d7d441324330b20dc0395d97ccbc] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp, totalSize=36.4 K 2024-11-10T03:17:25,436 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8e6259fdb1df4cf99e5fd6801c7b1c4c, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731208641954 2024-11-10T03:17:25,437 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 78598c9ae8dd487dacf716cad4a34c56, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1731208642340 2024-11-10T03:17:25,437 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4241d7d441324330b20dc0395d97ccbc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1731208644493 2024-11-10T03:17:25,450 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a93817da82260912633bd4e3d359d5b7#C#compaction#251 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:17:25,451 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/C/a6a7335e58b541aeaf5063373ed6d989 is 50, key is test_row_0/C:col10/1731208644493/Put/seqid=0 2024-11-10T03:17:25,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742124_1300 (size=12915) 2024-11-10T03:17:25,468 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/C/a6a7335e58b541aeaf5063373ed6d989 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/a6a7335e58b541aeaf5063373ed6d989 2024-11-10T03:17:25,476 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a93817da82260912633bd4e3d359d5b7/C of a93817da82260912633bd4e3d359d5b7 into a6a7335e58b541aeaf5063373ed6d989(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:17:25,476 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a93817da82260912633bd4e3d359d5b7: 2024-11-10T03:17:25,476 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., storeName=a93817da82260912633bd4e3d359d5b7/C, priority=13, startTime=1731208645386; duration=0sec 2024-11-10T03:17:25,476 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:17:25,476 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a93817da82260912633bd4e3d359d5b7:C 2024-11-10T03:17:25,504 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-10T03:17:25,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:25,504 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing a93817da82260912633bd4e3d359d5b7 3/3 column families, dataSize=131.94 KB heapSize=365.84 KB 2024-11-10T03:17:25,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a93817da82260912633bd4e3d359d5b7, store=A 2024-11-10T03:17:25,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T03:17:25,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a93817da82260912633bd4e3d359d5b7, store=B 2024-11-10T03:17:25,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T03:17:25,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a93817da82260912633bd4e3d359d5b7, store=C 2024-11-10T03:17:25,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T03:17:25,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/A/284a31358cd14c2fb6f93b91956cb98c is 50, key is test_row_0/A:col10/1731208644565/Put/seqid=0 2024-11-10T03:17:25,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742125_1301 (size=12301) 2024-11-10T03:17:25,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on a93817da82260912633bd4e3d359d5b7 2024-11-10T03:17:25,715 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. as already flushing 2024-11-10T03:17:25,731 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:25,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49990 deadline: 1731208705726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:25,731 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:25,731 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:25,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49966 deadline: 1731208705727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:25,731 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:25,731 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:25,731 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:25,731 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:25,732 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:25,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50002 deadline: 1731208705727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:25,732 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:25,732 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:25,732 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:25,732 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:25,732 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:25,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49936 deadline: 1731208705728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:25,732 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:25,732 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:25,732 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:25,735 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:25,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49950 deadline: 1731208705730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:25,735 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:25,735 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:25,735 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:25,827 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/B/8f0f041344c34abeb0cbaa906ef503af as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/8f0f041344c34abeb0cbaa906ef503af 2024-11-10T03:17:25,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:25,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49966 deadline: 1731208705836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:25,837 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:25,837 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:25,838 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:25,844 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:25,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49990 deadline: 1731208705844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:25,845 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:25,845 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:25,845 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:25,845 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:25,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49936 deadline: 1731208705844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:25,846 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:25,846 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:25,846 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:25,846 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:25,846 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:25,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50002 deadline: 1731208705844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:25,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49950 deadline: 1731208705844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:25,846 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:25,846 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:25,847 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:25,847 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:25,847 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:25,847 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:25,850 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a93817da82260912633bd4e3d359d5b7/B of a93817da82260912633bd4e3d359d5b7 into 8f0f041344c34abeb0cbaa906ef503af(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:17:25,850 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a93817da82260912633bd4e3d359d5b7: 2024-11-10T03:17:25,850 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., storeName=a93817da82260912633bd4e3d359d5b7/B, priority=13, startTime=1731208645386; duration=0sec 2024-11-10T03:17:25,850 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:17:25,851 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a93817da82260912633bd4e3d359d5b7:B 2024-11-10T03:17:25,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-10T03:17:25,922 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/A/284a31358cd14c2fb6f93b91956cb98c 2024-11-10T03:17:25,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/B/001d0cdf4edc4ef58b1aabfb190f11a5 is 50, key is test_row_0/B:col10/1731208644565/Put/seqid=0 2024-11-10T03:17:25,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742126_1302 (size=12301) 2024-11-10T03:17:26,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:26,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49966 deadline: 1731208706042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:26,044 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:26,044 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:26,045 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:26,055 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:26,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49990 deadline: 1731208706053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:26,055 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:26,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:26,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:26,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:26,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50002 deadline: 1731208706053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:26,056 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:26,056 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:26,056 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:26,056 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:26,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49936 deadline: 1731208706053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:26,056 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:26,056 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:26,056 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:26,056 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:26,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49950 deadline: 1731208706053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:26,056 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:26,056 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:26,056 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:26,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:26,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49966 deadline: 1731208706353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:26,357 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:26,357 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:26,357 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:26,357 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/B/001d0cdf4edc4ef58b1aabfb190f11a5 2024-11-10T03:17:26,364 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:26,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49936 deadline: 1731208706364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:26,364 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:26,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49990 deadline: 1731208706364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:26,365 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:26,365 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:26,365 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:26,365 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:26,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50002 deadline: 1731208706364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:26,365 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:26,365 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:26,365 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:26,365 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:26,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49950 deadline: 1731208706364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:26,365 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:26,365 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:26,365 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:26,366 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:26,366 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:26,366 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:26,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/C/6d98934f4c424e8383b335f13bf37384 is 50, key is test_row_0/C:col10/1731208644565/Put/seqid=0 2024-11-10T03:17:26,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742127_1303 (size=12301) 2024-11-10T03:17:26,429 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/C/6d98934f4c424e8383b335f13bf37384 2024-11-10T03:17:26,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/A/284a31358cd14c2fb6f93b91956cb98c as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/284a31358cd14c2fb6f93b91956cb98c 2024-11-10T03:17:26,471 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/284a31358cd14c2fb6f93b91956cb98c, entries=150, sequenceid=315, filesize=12.0 K 2024-11-10T03:17:26,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/B/001d0cdf4edc4ef58b1aabfb190f11a5 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/001d0cdf4edc4ef58b1aabfb190f11a5 2024-11-10T03:17:26,489 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/001d0cdf4edc4ef58b1aabfb190f11a5, entries=150, sequenceid=315, filesize=12.0 K 2024-11-10T03:17:26,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/C/6d98934f4c424e8383b335f13bf37384 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/6d98934f4c424e8383b335f13bf37384 2024-11-10T03:17:26,505 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/6d98934f4c424e8383b335f13bf37384, entries=150, sequenceid=315, filesize=12.0 K 2024-11-10T03:17:26,513 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~131.94 KB/135110, heapSize ~365.80 KB/374576, currentSize=53.67 KB/54960 for a93817da82260912633bd4e3d359d5b7 in 1009ms, sequenceid=315, compaction requested=false 2024-11-10T03:17:26,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for a93817da82260912633bd4e3d359d5b7: 2024-11-10T03:17:26,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:26,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-10T03:17:26,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-10T03:17:26,518 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-10T03:17:26,518 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7800 sec 2024-11-10T03:17:26,524 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.7860 sec 2024-11-10T03:17:26,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on a93817da82260912633bd4e3d359d5b7 2024-11-10T03:17:26,866 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a93817da82260912633bd4e3d359d5b7 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-10T03:17:26,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a93817da82260912633bd4e3d359d5b7, store=A 2024-11-10T03:17:26,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:17:26,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a93817da82260912633bd4e3d359d5b7, store=B 2024-11-10T03:17:26,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:17:26,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a93817da82260912633bd4e3d359d5b7, store=C 2024-11-10T03:17:26,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:17:26,871 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/A/b74669fda74f4fc5b7d19dc8b5fd88a1 is 50, key is test_row_0/A:col10/1731208645726/Put/seqid=0 2024-11-10T03:17:26,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742128_1304 (size=12301) 2024-11-10T03:17:26,892 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:26,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49966 deadline: 1731208706886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:26,893 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:26,893 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:26,893 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:26,893 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:26,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50002 deadline: 1731208706887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:26,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-10T03:17:26,893 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:26,893 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:26,893 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:26,893 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T03:17:26,894 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T03:17:26,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-10T03:17:26,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-10T03:17:26,895 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T03:17:26,895 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T03:17:26,896 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T03:17:26,896 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:26,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49950 deadline: 1731208706891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:26,896 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:26,896 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:26,896 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:26,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:26,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49936 deadline: 1731208706891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:26,899 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:26,899 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:26,899 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:26,899 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:26,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49990 deadline: 1731208706892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:26,900 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:26,900 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:26,900 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:27,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-10T03:17:27,006 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:27,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49966 deadline: 1731208707003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:27,006 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:27,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50002 deadline: 1731208707003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:27,006 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:27,006 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:27,007 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:27,007 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:27,007 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:27,007 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:27,007 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:27,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49950 deadline: 1731208707003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:27,007 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:27,007 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:27,007 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:27,007 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:27,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49936 deadline: 1731208707003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:27,007 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:27,007 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:27,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49990 deadline: 1731208707003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:27,007 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:27,007 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:27,007 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:27,007 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:27,007 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:27,048 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-10T03:17:27,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:27,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. as already flushing 2024-11-10T03:17:27,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:27,048 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:27,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:27,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:27,201 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-10T03:17:27,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:27,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. as already flushing 2024-11-10T03:17:27,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:27,201 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:27,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:27,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:27,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-10T03:17:27,213 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:27,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:27,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50002 deadline: 1731208707213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:27,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49950 deadline: 1731208707213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:27,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:27,214 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:27,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49966 deadline: 1731208707213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:27,214 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:27,214 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:27,214 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:27,215 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:27,215 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:27,215 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:27,215 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:27,215 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:27,226 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:27,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49936 deadline: 1731208707224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:27,226 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:27,226 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:27,226 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:27,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49990 deadline: 1731208707224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:27,226 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:27,226 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:27,226 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:27,226 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:27,275 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/A/b74669fda74f4fc5b7d19dc8b5fd88a1 2024-11-10T03:17:27,292 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/B/25ab8a1351f849398273bb49e0882d65 is 50, key is test_row_0/B:col10/1731208645726/Put/seqid=0 2024-11-10T03:17:27,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742129_1305 (size=12301) 2024-11-10T03:17:27,312 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/B/25ab8a1351f849398273bb49e0882d65 2024-11-10T03:17:27,330 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/C/dd5db3dc21504538b16f24558a94f7c2 is 50, key is test_row_0/C:col10/1731208645726/Put/seqid=0 2024-11-10T03:17:27,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742130_1306 (size=12301) 2024-11-10T03:17:27,352 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/C/dd5db3dc21504538b16f24558a94f7c2 2024-11-10T03:17:27,355 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-10T03:17:27,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:27,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. as already flushing 2024-11-10T03:17:27,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:27,355 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:27,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:27,359 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/A/b74669fda74f4fc5b7d19dc8b5fd88a1 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/b74669fda74f4fc5b7d19dc8b5fd88a1 2024-11-10T03:17:27,367 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/b74669fda74f4fc5b7d19dc8b5fd88a1, entries=150, sequenceid=328, filesize=12.0 K 2024-11-10T03:17:27,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:27,368 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/B/25ab8a1351f849398273bb49e0882d65 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/25ab8a1351f849398273bb49e0882d65 2024-11-10T03:17:27,374 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/25ab8a1351f849398273bb49e0882d65, entries=150, sequenceid=328, filesize=12.0 K 2024-11-10T03:17:27,376 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/C/dd5db3dc21504538b16f24558a94f7c2 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/dd5db3dc21504538b16f24558a94f7c2 2024-11-10T03:17:27,384 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/dd5db3dc21504538b16f24558a94f7c2, entries=150, sequenceid=328, filesize=12.0 K 2024-11-10T03:17:27,385 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for a93817da82260912633bd4e3d359d5b7 in 519ms, sequenceid=328, compaction requested=true 2024-11-10T03:17:27,385 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a93817da82260912633bd4e3d359d5b7: 2024-11-10T03:17:27,385 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T03:17:27,385 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a93817da82260912633bd4e3d359d5b7:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T03:17:27,385 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:17:27,385 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T03:17:27,386 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T03:17:27,386 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1541): a93817da82260912633bd4e3d359d5b7/B is initiating minor compaction (all files) 2024-11-10T03:17:27,386 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a93817da82260912633bd4e3d359d5b7/B in TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:27,386 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/8f0f041344c34abeb0cbaa906ef503af, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/001d0cdf4edc4ef58b1aabfb190f11a5, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/25ab8a1351f849398273bb49e0882d65] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp, totalSize=36.6 K 2024-11-10T03:17:27,387 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T03:17:27,387 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1541): a93817da82260912633bd4e3d359d5b7/A is initiating minor compaction (all files) 2024-11-10T03:17:27,387 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a93817da82260912633bd4e3d359d5b7/A in TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:27,387 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/58a2785993274f8295c8e084d40e97eb, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/284a31358cd14c2fb6f93b91956cb98c, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/b74669fda74f4fc5b7d19dc8b5fd88a1] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp, totalSize=36.6 K 2024-11-10T03:17:27,387 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 58a2785993274f8295c8e084d40e97eb, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1731208644493 2024-11-10T03:17:27,387 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f0f041344c34abeb0cbaa906ef503af, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1731208644493 2024-11-10T03:17:27,388 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 284a31358cd14c2fb6f93b91956cb98c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731208644565 2024-11-10T03:17:27,388 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 001d0cdf4edc4ef58b1aabfb190f11a5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731208644565 2024-11-10T03:17:27,388 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting b74669fda74f4fc5b7d19dc8b5fd88a1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731208645726 2024-11-10T03:17:27,388 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a93817da82260912633bd4e3d359d5b7:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T03:17:27,388 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 25ab8a1351f849398273bb49e0882d65, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731208645726 2024-11-10T03:17:27,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:17:27,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a93817da82260912633bd4e3d359d5b7:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T03:17:27,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T03:17:27,408 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a93817da82260912633bd4e3d359d5b7#B#compaction#258 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:17:27,409 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/B/ec8a659fda5444278b34069211fd5e93 is 50, key is test_row_0/B:col10/1731208645726/Put/seqid=0 2024-11-10T03:17:27,411 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a93817da82260912633bd4e3d359d5b7#A#compaction#259 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:17:27,411 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/A/29178a79482a41c893f5c24c433d4c5a is 50, key is test_row_0/A:col10/1731208645726/Put/seqid=0 2024-11-10T03:17:27,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742131_1307 (size=13017) 2024-11-10T03:17:27,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742132_1308 (size=13017) 2024-11-10T03:17:27,450 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/B/ec8a659fda5444278b34069211fd5e93 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/ec8a659fda5444278b34069211fd5e93 2024-11-10T03:17:27,456 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a93817da82260912633bd4e3d359d5b7/B of a93817da82260912633bd4e3d359d5b7 into ec8a659fda5444278b34069211fd5e93(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:17:27,456 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a93817da82260912633bd4e3d359d5b7: 2024-11-10T03:17:27,456 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., storeName=a93817da82260912633bd4e3d359d5b7/B, priority=13, startTime=1731208647385; duration=0sec 2024-11-10T03:17:27,456 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T03:17:27,456 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a93817da82260912633bd4e3d359d5b7:B 2024-11-10T03:17:27,456 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T03:17:27,458 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T03:17:27,458 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1541): a93817da82260912633bd4e3d359d5b7/C is initiating minor compaction (all files) 2024-11-10T03:17:27,458 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a93817da82260912633bd4e3d359d5b7/C in TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:27,458 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/a6a7335e58b541aeaf5063373ed6d989, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/6d98934f4c424e8383b335f13bf37384, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/dd5db3dc21504538b16f24558a94f7c2] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp, totalSize=36.6 K 2024-11-10T03:17:27,459 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting a6a7335e58b541aeaf5063373ed6d989, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1731208644493 2024-11-10T03:17:27,459 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d98934f4c424e8383b335f13bf37384, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731208644565 2024-11-10T03:17:27,459 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting dd5db3dc21504538b16f24558a94f7c2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731208645726 2024-11-10T03:17:27,482 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a93817da82260912633bd4e3d359d5b7#C#compaction#260 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:17:27,482 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/C/403e954d5ba049beb30576a4926d3ea6 is 50, key is test_row_0/C:col10/1731208645726/Put/seqid=0 2024-11-10T03:17:27,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742133_1309 (size=13017) 2024-11-10T03:17:27,492 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/C/403e954d5ba049beb30576a4926d3ea6 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/403e954d5ba049beb30576a4926d3ea6 2024-11-10T03:17:27,498 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a93817da82260912633bd4e3d359d5b7/C of a93817da82260912633bd4e3d359d5b7 into 403e954d5ba049beb30576a4926d3ea6(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:17:27,498 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a93817da82260912633bd4e3d359d5b7: 2024-11-10T03:17:27,498 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., storeName=a93817da82260912633bd4e3d359d5b7/C, priority=13, startTime=1731208647392; duration=0sec 2024-11-10T03:17:27,498 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:17:27,498 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a93817da82260912633bd4e3d359d5b7:C 2024-11-10T03:17:27,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-10T03:17:27,525 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-10T03:17:27,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:27,526 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing a93817da82260912633bd4e3d359d5b7 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-10T03:17:27,528 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731208646889, max timestamp=1731208647524Num uniques 859; 2024-11-10T03:17:27,528 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T03:17:27,531 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731208646889, max timestamp=1731208647524Num uniques 859; 2024-11-10T03:17:27,531 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T03:17:27,531 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731208646889, max timestamp=1731208647524Num uniques 859; 2024-11-10T03:17:27,531 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T03:17:27,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a93817da82260912633bd4e3d359d5b7, store=A 2024-11-10T03:17:27,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T03:17:27,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a93817da82260912633bd4e3d359d5b7, store=B 2024-11-10T03:17:27,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T03:17:27,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a93817da82260912633bd4e3d359d5b7, store=C 2024-11-10T03:17:27,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T03:17:27,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/A/12b2750234504d5e9292e2dc3595bb61 is 50, key is test_row_0/A:col10/1731208646891/Put/seqid=0 2024-11-10T03:17:27,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on a93817da82260912633bd4e3d359d5b7 2024-11-10T03:17:27,541 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. as already flushing 2024-11-10T03:17:27,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742134_1310 (size=12301) 2024-11-10T03:17:27,571 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731208647549, max timestamp=1731208647567Num uniques 859; 2024-11-10T03:17:27,571 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T03:17:27,571 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731208647549, max timestamp=1731208647567Num uniques 859; 2024-11-10T03:17:27,571 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T03:17:27,573 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731208647549, max timestamp=1731208647567Num uniques 859; 2024-11-10T03:17:27,573 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-10T03:17:27,608 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731208647594, max timestamp=1731208647603Num uniques 1018; 2024-11-10T03:17:27,608 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731208647594, max timestamp=1731208647603Num uniques 1018; 2024-11-10T03:17:27,608 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-10T03:17:27,608 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-10T03:17:27,611 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731208647594, max timestamp=1731208647604Num uniques 1009; 2024-11-10T03:17:27,611 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46213, new segment data size=9978, suffix heap size=132904,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=218 2024-11-10T03:17:27,644 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731208647627, max timestamp=1731208647638Num uniques 1077; 2024-11-10T03:17:27,644 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-10T03:17:27,647 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731208647627, max timestamp=1731208647638Num uniques 1077; 2024-11-10T03:17:27,647 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731208647627, max timestamp=1731208647638Num uniques 1077; 2024-11-10T03:17:27,647 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-11-10T03:17:27,647 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-11-10T03:17:27,694 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731208647677, max timestamp=1731208647690Num uniques 1036; 2024-11-10T03:17:27,694 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=10806, suffix heap size=135712,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-11-10T03:17:27,694 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731208647677, max timestamp=1731208647690Num uniques 1086; 2024-11-10T03:17:27,694 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-11-10T03:17:27,694 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731208647677, max timestamp=1731208647690Num uniques 1036; 2024-11-10T03:17:27,694 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=10806, suffix heap size=135712,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-11-10T03:17:27,732 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=345, cellSize=15800, totalHeapSize=36120, min timestamp=1731208647708, max timestamp=1731208647728Num uniques 1095; 2024-11-10T03:17:27,733 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=15800, suffix heap size=141848,new segment heap size=36120 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=345 2024-11-10T03:17:27,733 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=345, cellSize=15800, totalHeapSize=36120, min timestamp=1731208647708, max timestamp=1731208647728Num uniques 1095; 2024-11-10T03:17:27,733 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=15800, suffix heap size=141848,new segment heap size=36120 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=345 2024-11-10T03:17:27,733 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=345, cellSize=15800, totalHeapSize=36120, min timestamp=1731208647708, max timestamp=1731208647728Num uniques 1095; 2024-11-10T03:17:27,733 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=15800, suffix heap size=141848,new segment heap size=36120 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=345 2024-11-10T03:17:27,764 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:27,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50002 deadline: 1731208707757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:27,764 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:27,764 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:27,764 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:27,770 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:27,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49966 deadline: 1731208707762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:27,770 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:27,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49936 deadline: 1731208707762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:27,771 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:27,771 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:27,771 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:27,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49990 deadline: 1731208707763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:27,771 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:27,771 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:27,771 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:27,771 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:27,771 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:27,771 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:27,771 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:27,772 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:27,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49950 deadline: 1731208707765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:27,772 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:27,772 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:27,772 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:27,852 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/A/29178a79482a41c893f5c24c433d4c5a as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/29178a79482a41c893f5c24c433d4c5a 2024-11-10T03:17:27,866 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a93817da82260912633bd4e3d359d5b7/A of a93817da82260912633bd4e3d359d5b7 into 29178a79482a41c893f5c24c433d4c5a(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:17:27,866 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a93817da82260912633bd4e3d359d5b7: 2024-11-10T03:17:27,866 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., storeName=a93817da82260912633bd4e3d359d5b7/A, priority=13, startTime=1731208647385; duration=0sec 2024-11-10T03:17:27,866 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:17:27,866 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a93817da82260912633bd4e3d359d5b7:A 2024-11-10T03:17:27,880 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:27,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49966 deadline: 1731208707874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:27,881 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:27,881 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:27,881 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:27,881 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:27,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50002 deadline: 1731208707874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:27,881 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:27,881 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:27,881 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:27,881 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:27,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49936 deadline: 1731208707874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:27,882 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:27,882 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:27,882 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:27,887 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:27,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49950 deadline: 1731208707883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:27,888 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:27,888 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:27,888 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:27,888 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:27,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49990 deadline: 1731208707883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:27,888 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:27,888 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:27,888 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:27,952 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/A/12b2750234504d5e9292e2dc3595bb61 2024-11-10T03:17:27,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/B/2ad18bfc772749fc9dfcbaa37c280f27 is 50, key is test_row_0/B:col10/1731208646891/Put/seqid=0 2024-11-10T03:17:28,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742135_1311 (size=12301) 2024-11-10T03:17:28,016 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/B/2ad18bfc772749fc9dfcbaa37c280f27 2024-11-10T03:17:28,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-10T03:17:28,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/C/43dc48163c1c4ca3bdb0150bf5db3220 is 50, key is test_row_0/C:col10/1731208646891/Put/seqid=0 2024-11-10T03:17:28,099 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:28,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49966 deadline: 1731208708094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:28,100 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:28,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50002 deadline: 1731208708094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:28,101 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:28,101 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:28,101 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:28,101 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:28,101 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:28,101 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:28,101 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:28,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49936 deadline: 1731208708094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:28,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:28,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49950 deadline: 1731208708094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:28,102 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:28,102 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:28,102 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:28,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49990 deadline: 1731208708094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:28,102 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:28,102 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:28,102 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:28,102 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:28,102 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:28,102 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:28,102 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:28,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742136_1312 (size=12301) 2024-11-10T03:17:28,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:28,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50002 deadline: 1731208708403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:28,413 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:28,413 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:28,413 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:28,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:28,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49990 deadline: 1731208708413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:28,420 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:28,420 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:28,420 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:28,426 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:28,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49966 deadline: 1731208708416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:28,427 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:28,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49950 deadline: 1731208708418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:28,428 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:28,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49936 deadline: 1731208708418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:28,428 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:28,428 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:28,428 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:28,428 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:28,428 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:28,428 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:28,432 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:28,433 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:28,433 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:28,514 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/C/43dc48163c1c4ca3bdb0150bf5db3220 2024-11-10T03:17:28,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/A/12b2750234504d5e9292e2dc3595bb61 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/12b2750234504d5e9292e2dc3595bb61 2024-11-10T03:17:28,569 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/12b2750234504d5e9292e2dc3595bb61, entries=150, sequenceid=351, filesize=12.0 K 2024-11-10T03:17:28,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/B/2ad18bfc772749fc9dfcbaa37c280f27 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/2ad18bfc772749fc9dfcbaa37c280f27 2024-11-10T03:17:28,593 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/2ad18bfc772749fc9dfcbaa37c280f27, entries=150, sequenceid=351, filesize=12.0 K 2024-11-10T03:17:28,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/C/43dc48163c1c4ca3bdb0150bf5db3220 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/43dc48163c1c4ca3bdb0150bf5db3220 2024-11-10T03:17:28,621 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/43dc48163c1c4ca3bdb0150bf5db3220, entries=150, sequenceid=351, filesize=12.0 K 2024-11-10T03:17:28,623 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~66.19 KB/67776, currentSize=161.02 KB/164880 for a93817da82260912633bd4e3d359d5b7 in 1097ms, sequenceid=351, compaction requested=false 2024-11-10T03:17:28,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for a93817da82260912633bd4e3d359d5b7: 2024-11-10T03:17:28,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:28,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-10T03:17:28,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-10T03:17:28,638 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-10T03:17:28,638 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7350 sec 2024-11-10T03:17:28,642 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.7460 sec 2024-11-10T03:17:28,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on a93817da82260912633bd4e3d359d5b7 2024-11-10T03:17:28,926 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a93817da82260912633bd4e3d359d5b7 3/3 column families, dataSize=167.72 KB heapSize=467.86 KB 2024-11-10T03:17:28,927 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a93817da82260912633bd4e3d359d5b7, store=A 2024-11-10T03:17:28,927 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-10T03:17:28,927 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a93817da82260912633bd4e3d359d5b7, store=B 2024-11-10T03:17:28,927 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-10T03:17:28,927 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a93817da82260912633bd4e3d359d5b7, store=C 2024-11-10T03:17:28,927 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-10T03:17:28,938 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/A/fd5d50b58eab4acc846c2c56e7588b17 is 50, key is test_row_0/A:col10/1731208647759/Put/seqid=0 2024-11-10T03:17:28,948 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:28,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50002 deadline: 1731208708938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:28,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:28,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49950 deadline: 1731208708942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:28,950 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:28,950 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:28,950 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:28,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:28,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49966 deadline: 1731208708943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:28,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:28,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49936 deadline: 1731208708944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:28,951 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:28,951 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:28,951 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:28,951 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:28,951 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:28,951 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:28,951 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:28,952 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:28,952 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:28,953 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:28,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49990 deadline: 1731208708947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:28,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:28,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:28,957 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:28,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742137_1313 (size=14741) 2024-11-10T03:17:28,983 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=459 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/A/fd5d50b58eab4acc846c2c56e7588b17 2024-11-10T03:17:29,004 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/B/05d723fef028446a83c6a3ca1f064f4b is 50, key is test_row_0/B:col10/1731208647759/Put/seqid=0 2024-11-10T03:17:29,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-10T03:17:29,044 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T03:17:29,044 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T03:17:29,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-10T03:17:29,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742138_1314 (size=12301) 2024-11-10T03:17:29,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-10T03:17:29,050 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=459 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/B/05d723fef028446a83c6a3ca1f064f4b 2024-11-10T03:17:29,052 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T03:17:29,054 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T03:17:29,054 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T03:17:29,061 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:29,061 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:29,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49936 deadline: 1731208709054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:29,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50002 deadline: 1731208709053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:29,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:29,062 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:29,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:29,062 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:29,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:29,062 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:29,069 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:29,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49950 deadline: 1731208709063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:29,069 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:29,070 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:29,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49990 deadline: 1731208709063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:29,070 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:29,070 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:29,072 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/C/0d09f75fa7b4436daf3dcc9a59112396 is 50, key is test_row_0/C:col10/1731208647759/Put/seqid=0 2024-11-10T03:17:29,074 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:29,074 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:29,074 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:29,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742139_1315 (size=12301) 2024-11-10T03:17:29,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-10T03:17:29,206 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-10T03:17:29,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:29,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. as already flushing 2024-11-10T03:17:29,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:29,210 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:29,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:29,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:29,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:29,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50002 deadline: 1731208709274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:29,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:29,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49950 deadline: 1731208709274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:29,277 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:29,277 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:29,278 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:29,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:29,278 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:29,278 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:29,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49936 deadline: 1731208709274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:29,278 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:29,278 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:29,278 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:29,278 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:29,286 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:29,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49990 deadline: 1731208709283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:29,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:29,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:29,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:29,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-10T03:17:29,372 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-10T03:17:29,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:29,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. as already flushing 2024-11-10T03:17:29,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:29,372 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:29,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:29,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:29,504 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=459 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/C/0d09f75fa7b4436daf3dcc9a59112396 2024-11-10T03:17:29,525 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/A/fd5d50b58eab4acc846c2c56e7588b17 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/fd5d50b58eab4acc846c2c56e7588b17 2024-11-10T03:17:29,534 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-10T03:17:29,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:29,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. as already flushing 2024-11-10T03:17:29,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:29,535 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:29,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:29,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:29,536 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/fd5d50b58eab4acc846c2c56e7588b17, entries=200, sequenceid=459, filesize=14.4 K 2024-11-10T03:17:29,539 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/B/05d723fef028446a83c6a3ca1f064f4b as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/05d723fef028446a83c6a3ca1f064f4b 2024-11-10T03:17:29,549 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/05d723fef028446a83c6a3ca1f064f4b, entries=150, sequenceid=459, filesize=12.0 K 2024-11-10T03:17:29,552 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/C/0d09f75fa7b4436daf3dcc9a59112396 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/0d09f75fa7b4436daf3dcc9a59112396 2024-11-10T03:17:29,573 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/0d09f75fa7b4436daf3dcc9a59112396, entries=150, sequenceid=459, filesize=12.0 K 2024-11-10T03:17:29,577 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~467.81 KB/479040, currentSize=26.84 KB/27480 for a93817da82260912633bd4e3d359d5b7 in 651ms, sequenceid=459, compaction requested=true 2024-11-10T03:17:29,577 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a93817da82260912633bd4e3d359d5b7: 2024-11-10T03:17:29,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a93817da82260912633bd4e3d359d5b7:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T03:17:29,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T03:17:29,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a93817da82260912633bd4e3d359d5b7:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T03:17:29,578 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-10T03:17:29,578 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a93817da82260912633bd4e3d359d5b7:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T03:17:29,578 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-10T03:17:29,580 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T03:17:29,580 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T03:17:29,581 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T03:17:29,581 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1541): a93817da82260912633bd4e3d359d5b7/C is initiating minor compaction (all files) 2024-11-10T03:17:29,581 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a93817da82260912633bd4e3d359d5b7/C in TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:29,581 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/403e954d5ba049beb30576a4926d3ea6, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/43dc48163c1c4ca3bdb0150bf5db3220, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/0d09f75fa7b4436daf3dcc9a59112396] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp, totalSize=36.7 K 2024-11-10T03:17:29,581 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40059 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T03:17:29,582 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1541): a93817da82260912633bd4e3d359d5b7/A is initiating minor compaction (all files) 2024-11-10T03:17:29,582 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a93817da82260912633bd4e3d359d5b7/A in TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:29,582 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 403e954d5ba049beb30576a4926d3ea6, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731208645726 2024-11-10T03:17:29,582 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/29178a79482a41c893f5c24c433d4c5a, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/12b2750234504d5e9292e2dc3595bb61, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/fd5d50b58eab4acc846c2c56e7588b17] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp, totalSize=39.1 K 2024-11-10T03:17:29,582 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 43dc48163c1c4ca3bdb0150bf5db3220, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1731208646889 2024-11-10T03:17:29,582 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0d09f75fa7b4436daf3dcc9a59112396, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=459, earliestPutTs=1731208647759 2024-11-10T03:17:29,582 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 29178a79482a41c893f5c24c433d4c5a, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731208645726 2024-11-10T03:17:29,583 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 12b2750234504d5e9292e2dc3595bb61, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1731208646889 2024-11-10T03:17:29,583 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting fd5d50b58eab4acc846c2c56e7588b17, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=459, earliestPutTs=1731208647759 2024-11-10T03:17:29,591 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a93817da82260912633bd4e3d359d5b7 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-10T03:17:29,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a93817da82260912633bd4e3d359d5b7, store=A 2024-11-10T03:17:29,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:17:29,592 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a93817da82260912633bd4e3d359d5b7, store=B 2024-11-10T03:17:29,592 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:17:29,592 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a93817da82260912633bd4e3d359d5b7, store=C 2024-11-10T03:17:29,592 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:17:29,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on a93817da82260912633bd4e3d359d5b7 2024-11-10T03:17:29,596 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a93817da82260912633bd4e3d359d5b7#C#compaction#267 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:17:29,597 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/C/9d716c96e64d4439958d4cf417ce2581 is 50, key is test_row_0/C:col10/1731208647759/Put/seqid=0 2024-11-10T03:17:29,598 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/A/dea7f65d96254628a297b478c8810980 is 50, key is test_row_0/A:col10/1731208649587/Put/seqid=0 2024-11-10T03:17:29,602 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a93817da82260912633bd4e3d359d5b7#A#compaction#269 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:17:29,603 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/A/738566be993541d6b66e6cc4493f0f5a is 50, key is test_row_0/A:col10/1731208647759/Put/seqid=0 2024-11-10T03:17:29,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742141_1317 (size=17181) 2024-11-10T03:17:29,668 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731208649646, max timestamp=1731208649662Num uniques 859; 2024-11-10T03:17:29,668 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T03:17:29,670 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731208649646, max timestamp=1731208649662Num uniques 859; 2024-11-10T03:17:29,670 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-10T03:17:29,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-10T03:17:29,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742142_1318 (size=13119) 2024-11-10T03:17:29,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742140_1316 (size=13119) 2024-11-10T03:17:29,688 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-10T03:17:29,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:29,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. as already flushing 2024-11-10T03:17:29,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:29,689 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:29,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:29,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:29,708 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/C/9d716c96e64d4439958d4cf417ce2581 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/9d716c96e64d4439958d4cf417ce2581 2024-11-10T03:17:29,711 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:29,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50002 deadline: 1731208709699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:29,712 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:29,712 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:29,712 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:29,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:29,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49950 deadline: 1731208709701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:29,716 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:29,716 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:29,716 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:29,716 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/A/738566be993541d6b66e6cc4493f0f5a as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/738566be993541d6b66e6cc4493f0f5a 2024-11-10T03:17:29,717 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:29,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49936 deadline: 1731208709706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:29,718 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:29,718 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:29,718 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:29,718 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:29,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49990 deadline: 1731208709713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:29,719 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:29,719 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:29,719 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:29,745 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a93817da82260912633bd4e3d359d5b7/A of a93817da82260912633bd4e3d359d5b7 into 738566be993541d6b66e6cc4493f0f5a(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:17:29,745 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a93817da82260912633bd4e3d359d5b7: 2024-11-10T03:17:29,745 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., storeName=a93817da82260912633bd4e3d359d5b7/A, priority=13, startTime=1731208649577; duration=0sec 2024-11-10T03:17:29,745 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T03:17:29,745 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a93817da82260912633bd4e3d359d5b7:A 2024-11-10T03:17:29,745 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T03:17:29,751 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T03:17:29,751 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1541): a93817da82260912633bd4e3d359d5b7/B is initiating minor compaction (all files) 2024-11-10T03:17:29,751 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a93817da82260912633bd4e3d359d5b7/B in TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:29,751 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/ec8a659fda5444278b34069211fd5e93, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/2ad18bfc772749fc9dfcbaa37c280f27, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/05d723fef028446a83c6a3ca1f064f4b] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp, totalSize=36.7 K 2024-11-10T03:17:29,752 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting ec8a659fda5444278b34069211fd5e93, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731208645726 2024-11-10T03:17:29,753 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a93817da82260912633bd4e3d359d5b7/C of a93817da82260912633bd4e3d359d5b7 into 9d716c96e64d4439958d4cf417ce2581(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:17:29,753 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a93817da82260912633bd4e3d359d5b7: 2024-11-10T03:17:29,753 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., storeName=a93817da82260912633bd4e3d359d5b7/C, priority=13, startTime=1731208649578; duration=0sec 2024-11-10T03:17:29,753 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:17:29,753 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a93817da82260912633bd4e3d359d5b7:C 2024-11-10T03:17:29,753 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ad18bfc772749fc9dfcbaa37c280f27, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1731208646889 2024-11-10T03:17:29,755 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 05d723fef028446a83c6a3ca1f064f4b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=459, earliestPutTs=1731208647759 2024-11-10T03:17:29,774 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a93817da82260912633bd4e3d359d5b7#B#compaction#270 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:17:29,775 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/B/f0c3d03b1558419ebcc87e4726174ef1 is 50, key is test_row_0/B:col10/1731208647759/Put/seqid=0 2024-11-10T03:17:29,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:29,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49950 deadline: 1731208709823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:29,828 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:29,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:29,828 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:29,828 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:29,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50002 deadline: 1731208709823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:29,829 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:29,829 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:29,829 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:29,829 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:29,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49936 deadline: 1731208709824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:29,830 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:29,830 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:29,830 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:29,830 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:29,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49990 deadline: 1731208709824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:29,830 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:29,830 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:29,830 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:29,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742143_1319 (size=13119) 2024-11-10T03:17:29,842 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-10T03:17:29,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:29,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. as already flushing 2024-11-10T03:17:29,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:29,843 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:29,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:29,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:29,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:29,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49966 deadline: 1731208709963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:29,967 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 394d2222f724,38139,1731208572054 for 'test_row_2' in a93817da82260912633bd4e3d359d5b7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T03:17:29,967 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:29,967 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:29,967 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:30,017 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-10T03:17:30,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:30,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. as already flushing 2024-11-10T03:17:30,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:30,020 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:30,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:30,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:30,035 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:30,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49950 deadline: 1731208710032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:30,035 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:30,035 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:30,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49936 deadline: 1731208710033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:30,035 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:30,035 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:30,035 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:30,035 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:30,035 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:30,035 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:30,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49990 deadline: 1731208710033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:30,036 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:30,036 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:30,036 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:30,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50002 deadline: 1731208710034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:30,036 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:30,036 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:30,036 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:30,036 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:30,048 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=469 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/A/dea7f65d96254628a297b478c8810980 2024-11-10T03:17:30,057 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/B/e8a34063b0774fc5b4d7d4a0fbf38532 is 50, key is test_row_0/B:col10/1731208649587/Put/seqid=0 2024-11-10T03:17:30,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742144_1320 (size=12301) 2024-11-10T03:17:30,175 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-10T03:17:30,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:30,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. as already flushing 2024-11-10T03:17:30,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:30,175 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:30,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:30,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:30,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-10T03:17:30,245 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/B/f0c3d03b1558419ebcc87e4726174ef1 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/f0c3d03b1558419ebcc87e4726174ef1 2024-11-10T03:17:30,250 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a93817da82260912633bd4e3d359d5b7/B of a93817da82260912633bd4e3d359d5b7 into f0c3d03b1558419ebcc87e4726174ef1(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:17:30,250 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a93817da82260912633bd4e3d359d5b7: 2024-11-10T03:17:30,250 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., storeName=a93817da82260912633bd4e3d359d5b7/B, priority=13, startTime=1731208649577; duration=0sec 2024-11-10T03:17:30,250 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:17:30,250 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a93817da82260912633bd4e3d359d5b7:B 2024-11-10T03:17:30,328 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-10T03:17:30,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:30,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. as already flushing 2024-11-10T03:17:30,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:30,328 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:30,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:30,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:30,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:30,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49950 deadline: 1731208710343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:30,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:30,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49936 deadline: 1731208710343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:30,345 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:30,345 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:30,345 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:30,345 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:30,345 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:30,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:30,345 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:30,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49990 deadline: 1731208710343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:30,346 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:30,346 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:30,346 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:30,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50002 deadline: 1731208710343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:30,346 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:30,346 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:30,346 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:30,346 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:30,474 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=469 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/B/e8a34063b0774fc5b4d7d4a0fbf38532 2024-11-10T03:17:30,481 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/C/34d96f46a1c041cab11dbaae559077fe is 50, key is test_row_0/C:col10/1731208649587/Put/seqid=0 2024-11-10T03:17:30,481 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-10T03:17:30,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:30,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. as already flushing 2024-11-10T03:17:30,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:30,481 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:30,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:30,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:30,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742145_1321 (size=12301) 2024-11-10T03:17:30,634 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-10T03:17:30,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:30,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. as already flushing 2024-11-10T03:17:30,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:30,635 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:30,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:30,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:30,793 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-10T03:17:30,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:30,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. as already flushing 2024-11-10T03:17:30,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:30,794 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:30,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:30,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:30,856 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:30,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49950 deadline: 1731208710853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:30,857 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:30,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49936 deadline: 1731208710853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:30,857 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:30,857 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:30,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:30,857 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:30,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49990 deadline: 1731208710853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:30,857 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:30,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50002 deadline: 1731208710853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:30,857 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:30,857 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:30,857 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:30,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:30,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:30,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:30,860 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:30,860 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:30,860 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:30,890 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=469 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/C/34d96f46a1c041cab11dbaae559077fe 2024-11-10T03:17:30,894 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/A/dea7f65d96254628a297b478c8810980 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/dea7f65d96254628a297b478c8810980 2024-11-10T03:17:30,900 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/dea7f65d96254628a297b478c8810980, entries=250, sequenceid=469, filesize=16.8 K 2024-11-10T03:17:30,901 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/B/e8a34063b0774fc5b4d7d4a0fbf38532 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/e8a34063b0774fc5b4d7d4a0fbf38532 2024-11-10T03:17:30,905 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/e8a34063b0774fc5b4d7d4a0fbf38532, entries=150, sequenceid=469, filesize=12.0 K 2024-11-10T03:17:30,907 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/C/34d96f46a1c041cab11dbaae559077fe as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/34d96f46a1c041cab11dbaae559077fe 2024-11-10T03:17:30,911 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/34d96f46a1c041cab11dbaae559077fe, entries=150, sequenceid=469, filesize=12.0 K 2024-11-10T03:17:30,912 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=131.55 KB/134705 for a93817da82260912633bd4e3d359d5b7 in 1321ms, sequenceid=469, compaction requested=false 2024-11-10T03:17:30,912 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a93817da82260912633bd4e3d359d5b7: 2024-11-10T03:17:30,951 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-10T03:17:30,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:30,951 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing a93817da82260912633bd4e3d359d5b7 3/3 column families, dataSize=131.55 KB heapSize=382.40 KB 2024-11-10T03:17:30,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a93817da82260912633bd4e3d359d5b7, store=A 2024-11-10T03:17:30,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T03:17:30,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a93817da82260912633bd4e3d359d5b7, store=B 2024-11-10T03:17:30,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T03:17:30,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a93817da82260912633bd4e3d359d5b7, store=C 2024-11-10T03:17:30,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T03:17:30,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/A/5381ee692eda44d0ab7f456711fa4202 is 50, key is test_row_0/A:col10/1731208649682/Put/seqid=0 2024-11-10T03:17:30,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742146_1322 (size=12301) 2024-11-10T03:17:30,992 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=504 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/A/5381ee692eda44d0ab7f456711fa4202 2024-11-10T03:17:31,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/B/122776a18a3447feb72d73844b534262 is 50, key is test_row_0/B:col10/1731208649682/Put/seqid=0 2024-11-10T03:17:31,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742147_1323 (size=12301) 2024-11-10T03:17:31,023 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.15 KB at sequenceid=504 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/B/122776a18a3447feb72d73844b534262 2024-11-10T03:17:31,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/C/2bab44195b0148198f3a2b005bb23384 is 50, key is test_row_0/C:col10/1731208649682/Put/seqid=0 2024-11-10T03:17:31,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742148_1324 (size=12301) 2024-11-10T03:17:31,047 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=64.85 KB at sequenceid=504 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/C/2bab44195b0148198f3a2b005bb23384 2024-11-10T03:17:31,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/A/5381ee692eda44d0ab7f456711fa4202 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/5381ee692eda44d0ab7f456711fa4202 2024-11-10T03:17:31,060 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/5381ee692eda44d0ab7f456711fa4202, entries=150, sequenceid=504, filesize=12.0 K 2024-11-10T03:17:31,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/B/122776a18a3447feb72d73844b534262 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/122776a18a3447feb72d73844b534262 2024-11-10T03:17:31,069 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/122776a18a3447feb72d73844b534262, entries=150, sequenceid=504, filesize=12.0 K 2024-11-10T03:17:31,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/C/2bab44195b0148198f3a2b005bb23384 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/2bab44195b0148198f3a2b005bb23384 2024-11-10T03:17:31,074 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/2bab44195b0148198f3a2b005bb23384, entries=150, sequenceid=504, filesize=12.0 K 2024-11-10T03:17:31,074 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~131.55 KB/134705, heapSize ~382.35 KB/391528, currentSize=0 B/0 for a93817da82260912633bd4e3d359d5b7 in 123ms, sequenceid=504, compaction requested=true 2024-11-10T03:17:31,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for a93817da82260912633bd4e3d359d5b7: 2024-11-10T03:17:31,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:31,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-10T03:17:31,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-10T03:17:31,077 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-10T03:17:31,077 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0210 sec 2024-11-10T03:17:31,078 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 2.0330 sec 2024-11-10T03:17:31,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-10T03:17:31,194 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T03:17:31,194 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T03:17:31,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-10T03:17:31,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-10T03:17:31,197 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T03:17:31,202 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T03:17:31,202 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T03:17:31,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-10T03:17:31,355 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-10T03:17:31,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:31,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for a93817da82260912633bd4e3d359d5b7: 2024-11-10T03:17:31,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:31,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-10T03:17:31,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-10T03:17:31,359 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-10T03:17:31,359 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 155 msec 2024-11-10T03:17:31,360 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 165 msec 2024-11-10T03:17:31,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-10T03:17:31,513 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T03:17:31,513 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T03:17:31,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-10T03:17:31,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-10T03:17:31,515 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T03:17:31,516 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T03:17:31,516 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T03:17:31,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-10T03:17:31,670 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-10T03:17:31,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:31,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for a93817da82260912633bd4e3d359d5b7: 2024-11-10T03:17:31,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:31,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-10T03:17:31,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-10T03:17:31,679 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-10T03:17:31,679 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 161 msec 2024-11-10T03:17:31,681 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 167 msec 2024-11-10T03:17:31,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-10T03:17:31,833 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T03:17:31,833 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T03:17:31,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-11-10T03:17:31,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-10T03:17:31,848 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T03:17:31,849 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T03:17:31,849 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T03:17:31,915 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a93817da82260912633bd4e3d359d5b7 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-10T03:17:31,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on a93817da82260912633bd4e3d359d5b7 2024-11-10T03:17:31,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a93817da82260912633bd4e3d359d5b7, store=A 2024-11-10T03:17:31,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:17:31,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a93817da82260912633bd4e3d359d5b7, store=B 2024-11-10T03:17:31,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:17:31,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a93817da82260912633bd4e3d359d5b7, store=C 2024-11-10T03:17:31,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:17:31,922 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/A/25ad4654f6f748a4a4ff026df459b659 is 50, key is test_row_0/A:col10/1731208651915/Put/seqid=0 2024-11-10T03:17:31,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742149_1325 (size=12301) 2024-11-10T03:17:31,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-10T03:17:31,966 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:31,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49936 deadline: 1731208711959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:31,967 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:31,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:31,967 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:31,967 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:31,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50002 deadline: 1731208711959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:31,967 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:31,967 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:31,967 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:31,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:31,967 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:31,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49950 deadline: 1731208711961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:31,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49990 deadline: 1731208711962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:31,967 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:31,968 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:31,968 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:31,968 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:31,968 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:31,968 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:31,977 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:31,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49966 deadline: 1731208711973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:31,978 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 394d2222f724,38139,1731208572054 for 'test_row_2' in a93817da82260912633bd4e3d359d5b7 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4216 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T03:17:31,978 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:31,978 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:31,978 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:32,003 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-10T03:17:32,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:32,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. as already flushing 2024-11-10T03:17:32,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:32,004 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:32,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:32,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:32,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:32,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49936 deadline: 1731208712073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:32,076 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:32,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:32,077 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:32,077 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:32,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50002 deadline: 1731208712073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:32,077 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:32,077 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:32,077 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:32,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49990 deadline: 1731208712073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:32,077 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:32,077 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:32,077 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:32,077 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:32,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49950 deadline: 1731208712073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:32,077 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:32,077 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:32,078 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:32,078 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:32,156 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-10T03:17:32,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:32,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. as already flushing 2024-11-10T03:17:32,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:32,156 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:32,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:32,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:32,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-10T03:17:32,284 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:32,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49936 deadline: 1731208712283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:32,284 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:32,284 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:32,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50002 deadline: 1731208712283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:32,284 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:32,284 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:32,284 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:32,284 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:32,284 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:32,284 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:32,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49990 deadline: 1731208712283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:32,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:32,285 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:32,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49950 deadline: 1731208712284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:32,285 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:32,285 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:32,285 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:32,285 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:32,285 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:32,309 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-10T03:17:32,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:32,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. as already flushing 2024-11-10T03:17:32,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:32,309 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:32,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:32,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:32,328 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=518 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/A/25ad4654f6f748a4a4ff026df459b659 2024-11-10T03:17:32,334 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/B/6eb89f4b4321476e81ca53d98fc25a7d is 50, key is test_row_0/B:col10/1731208651915/Put/seqid=0 2024-11-10T03:17:32,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742150_1326 (size=12301) 2024-11-10T03:17:32,462 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-10T03:17:32,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:32,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. as already flushing 2024-11-10T03:17:32,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:32,462 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:32,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:32,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:32,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-10T03:17:32,606 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:32,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49990 deadline: 1731208712593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:32,606 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:32,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49936 deadline: 1731208712593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:32,607 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:32,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50002 deadline: 1731208712593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:32,607 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:32,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49950 deadline: 1731208712603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:32,608 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:32,608 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:32,608 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:32,608 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:32,608 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:32,608 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:32,612 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:32,612 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:32,612 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:32,612 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:32,612 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:32,612 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:32,615 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-10T03:17:32,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:32,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. as already flushing 2024-11-10T03:17:32,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:32,615 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:32,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:32,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:32,750 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=518 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/B/6eb89f4b4321476e81ca53d98fc25a7d 2024-11-10T03:17:32,759 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/C/e9778bb4684547279efa6a5f8decef1c is 50, key is test_row_0/C:col10/1731208651915/Put/seqid=0 2024-11-10T03:17:32,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742151_1327 (size=12301) 2024-11-10T03:17:32,770 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-10T03:17:32,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:32,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. as already flushing 2024-11-10T03:17:32,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:32,770 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:32,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:32,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:32,922 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-10T03:17:32,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:32,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. as already flushing 2024-11-10T03:17:32,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:32,923 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:32,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:32,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:32,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-10T03:17:33,076 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-10T03:17:33,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:33,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. as already flushing 2024-11-10T03:17:33,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:33,076 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:33,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:33,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:33,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:33,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50002 deadline: 1731208713113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:33,117 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:33,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49950 deadline: 1731208713113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:33,117 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:33,117 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:33,117 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:33,117 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:33,117 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:33,117 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:33,126 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:33,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49990 deadline: 1731208713122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:33,126 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:33,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49936 deadline: 1731208713122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:33,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:33,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:33,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:33,126 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:33,127 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:33,127 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:33,163 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=518 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/C/e9778bb4684547279efa6a5f8decef1c 2024-11-10T03:17:33,168 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/A/25ad4654f6f748a4a4ff026df459b659 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/25ad4654f6f748a4a4ff026df459b659 2024-11-10T03:17:33,172 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/25ad4654f6f748a4a4ff026df459b659, entries=150, sequenceid=518, filesize=12.0 K 2024-11-10T03:17:33,174 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/B/6eb89f4b4321476e81ca53d98fc25a7d as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/6eb89f4b4321476e81ca53d98fc25a7d 2024-11-10T03:17:33,178 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/6eb89f4b4321476e81ca53d98fc25a7d, entries=150, sequenceid=518, filesize=12.0 K 2024-11-10T03:17:33,179 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/C/e9778bb4684547279efa6a5f8decef1c as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/e9778bb4684547279efa6a5f8decef1c 2024-11-10T03:17:33,184 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/e9778bb4684547279efa6a5f8decef1c, entries=150, sequenceid=518, filesize=12.0 K 2024-11-10T03:17:33,185 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for a93817da82260912633bd4e3d359d5b7 in 1270ms, sequenceid=518, compaction requested=true 2024-11-10T03:17:33,185 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a93817da82260912633bd4e3d359d5b7: 2024-11-10T03:17:33,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a93817da82260912633bd4e3d359d5b7:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T03:17:33,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:17:33,185 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T03:17:33,185 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T03:17:33,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a93817da82260912633bd4e3d359d5b7:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T03:17:33,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:17:33,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a93817da82260912633bd4e3d359d5b7:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T03:17:33,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T03:17:33,186 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50022 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T03:17:33,186 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1541): a93817da82260912633bd4e3d359d5b7/B is initiating minor compaction (all files) 2024-11-10T03:17:33,187 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a93817da82260912633bd4e3d359d5b7/B in TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:33,187 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/f0c3d03b1558419ebcc87e4726174ef1, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/e8a34063b0774fc5b4d7d4a0fbf38532, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/122776a18a3447feb72d73844b534262, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/6eb89f4b4321476e81ca53d98fc25a7d] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp, totalSize=48.8 K 2024-11-10T03:17:33,187 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54902 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T03:17:33,187 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1541): a93817da82260912633bd4e3d359d5b7/A is initiating minor compaction (all files) 2024-11-10T03:17:33,187 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a93817da82260912633bd4e3d359d5b7/A in TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:33,187 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/738566be993541d6b66e6cc4493f0f5a, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/dea7f65d96254628a297b478c8810980, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/5381ee692eda44d0ab7f456711fa4202, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/25ad4654f6f748a4a4ff026df459b659] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp, totalSize=53.6 K 2024-11-10T03:17:33,187 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting f0c3d03b1558419ebcc87e4726174ef1, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=459, earliestPutTs=1731208647759 2024-11-10T03:17:33,188 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 738566be993541d6b66e6cc4493f0f5a, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=459, earliestPutTs=1731208647759 2024-11-10T03:17:33,188 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting e8a34063b0774fc5b4d7d4a0fbf38532, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=469, earliestPutTs=1731208649586 2024-11-10T03:17:33,188 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 122776a18a3447feb72d73844b534262, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=504, earliestPutTs=1731208649682 2024-11-10T03:17:33,188 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting dea7f65d96254628a297b478c8810980, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=469, earliestPutTs=1731208648942 2024-11-10T03:17:33,188 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 6eb89f4b4321476e81ca53d98fc25a7d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=518, earliestPutTs=1731208651909 2024-11-10T03:17:33,189 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5381ee692eda44d0ab7f456711fa4202, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=504, earliestPutTs=1731208649682 2024-11-10T03:17:33,189 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 25ad4654f6f748a4a4ff026df459b659, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=518, earliestPutTs=1731208651909 2024-11-10T03:17:33,199 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a93817da82260912633bd4e3d359d5b7#A#compaction#279 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:17:33,201 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/A/8e401fbd1fb4479083f8583607e807fa is 50, key is test_row_0/A:col10/1731208651915/Put/seqid=0 2024-11-10T03:17:33,207 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a93817da82260912633bd4e3d359d5b7#B#compaction#280 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:17:33,207 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/B/edb61756398d4ebe8622960d7dc4f0ac is 50, key is test_row_0/B:col10/1731208651915/Put/seqid=0 2024-11-10T03:17:33,229 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-10T03:17:33,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:33,230 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing a93817da82260912633bd4e3d359d5b7 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-10T03:17:33,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a93817da82260912633bd4e3d359d5b7, store=A 2024-11-10T03:17:33,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:17:33,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a93817da82260912633bd4e3d359d5b7, store=B 2024-11-10T03:17:33,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:17:33,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a93817da82260912633bd4e3d359d5b7, store=C 2024-11-10T03:17:33,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:17:33,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/A/1939ba45854e436aac96c886293538bd is 50, key is test_row_0/A:col10/1731208651957/Put/seqid=0 2024-11-10T03:17:33,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742153_1329 (size=13255) 2024-11-10T03:17:33,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742152_1328 (size=13255) 2024-11-10T03:17:33,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742154_1330 (size=12301) 2024-11-10T03:17:33,245 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/B/edb61756398d4ebe8622960d7dc4f0ac as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/edb61756398d4ebe8622960d7dc4f0ac 2024-11-10T03:17:33,249 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a93817da82260912633bd4e3d359d5b7/B of a93817da82260912633bd4e3d359d5b7 into edb61756398d4ebe8622960d7dc4f0ac(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:17:33,249 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a93817da82260912633bd4e3d359d5b7: 2024-11-10T03:17:33,249 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., storeName=a93817da82260912633bd4e3d359d5b7/B, priority=12, startTime=1731208653185; duration=0sec 2024-11-10T03:17:33,250 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T03:17:33,250 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a93817da82260912633bd4e3d359d5b7:B 2024-11-10T03:17:33,250 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T03:17:33,252 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50022 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T03:17:33,252 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1541): a93817da82260912633bd4e3d359d5b7/C is initiating minor compaction (all files) 2024-11-10T03:17:33,252 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a93817da82260912633bd4e3d359d5b7/C in TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:33,252 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/9d716c96e64d4439958d4cf417ce2581, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/34d96f46a1c041cab11dbaae559077fe, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/2bab44195b0148198f3a2b005bb23384, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/e9778bb4684547279efa6a5f8decef1c] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp, totalSize=48.8 K 2024-11-10T03:17:33,253 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d716c96e64d4439958d4cf417ce2581, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=459, earliestPutTs=1731208647759 2024-11-10T03:17:33,253 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 34d96f46a1c041cab11dbaae559077fe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=469, earliestPutTs=1731208649586 2024-11-10T03:17:33,253 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 2bab44195b0148198f3a2b005bb23384, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=504, earliestPutTs=1731208649682 2024-11-10T03:17:33,254 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting e9778bb4684547279efa6a5f8decef1c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=518, earliestPutTs=1731208651909 2024-11-10T03:17:33,261 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a93817da82260912633bd4e3d359d5b7#C#compaction#282 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:17:33,261 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/C/8fc3599aedfc4ede9230e4fc55a92628 is 50, key is test_row_0/C:col10/1731208651915/Put/seqid=0 2024-11-10T03:17:33,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742155_1331 (size=13255) 2024-11-10T03:17:33,648 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=538 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/A/1939ba45854e436aac96c886293538bd 2024-11-10T03:17:33,649 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/A/8e401fbd1fb4479083f8583607e807fa as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/8e401fbd1fb4479083f8583607e807fa 2024-11-10T03:17:33,655 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a93817da82260912633bd4e3d359d5b7/A of a93817da82260912633bd4e3d359d5b7 into 8e401fbd1fb4479083f8583607e807fa(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:17:33,655 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a93817da82260912633bd4e3d359d5b7: 2024-11-10T03:17:33,655 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., storeName=a93817da82260912633bd4e3d359d5b7/A, priority=12, startTime=1731208653185; duration=0sec 2024-11-10T03:17:33,655 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:17:33,655 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a93817da82260912633bd4e3d359d5b7:A 2024-11-10T03:17:33,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/B/6a7b8515d6014d06bb354e589a5cd46d is 50, key is test_row_0/B:col10/1731208651957/Put/seqid=0 2024-11-10T03:17:33,682 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/C/8fc3599aedfc4ede9230e4fc55a92628 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/8fc3599aedfc4ede9230e4fc55a92628 2024-11-10T03:17:33,690 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a93817da82260912633bd4e3d359d5b7/C of a93817da82260912633bd4e3d359d5b7 into 8fc3599aedfc4ede9230e4fc55a92628(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:17:33,690 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a93817da82260912633bd4e3d359d5b7: 2024-11-10T03:17:33,690 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., storeName=a93817da82260912633bd4e3d359d5b7/C, priority=12, startTime=1731208653185; duration=0sec 2024-11-10T03:17:33,691 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:17:33,691 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a93817da82260912633bd4e3d359d5b7:C 2024-11-10T03:17:33,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742156_1332 (size=12301) 2024-11-10T03:17:33,701 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=538 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/B/6a7b8515d6014d06bb354e589a5cd46d 2024-11-10T03:17:33,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/C/5b015db3fcce437db83f465df162d21d is 50, key is test_row_0/C:col10/1731208651957/Put/seqid=0 2024-11-10T03:17:33,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742157_1333 (size=12301) 2024-11-10T03:17:33,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-10T03:17:34,118 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=538 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/C/5b015db3fcce437db83f465df162d21d 2024-11-10T03:17:34,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on a93817da82260912633bd4e3d359d5b7 2024-11-10T03:17:34,125 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. as already flushing 2024-11-10T03:17:34,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/A/1939ba45854e436aac96c886293538bd as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/1939ba45854e436aac96c886293538bd 2024-11-10T03:17:34,134 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/1939ba45854e436aac96c886293538bd, entries=150, sequenceid=538, filesize=12.0 K 2024-11-10T03:17:34,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/B/6a7b8515d6014d06bb354e589a5cd46d as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/6a7b8515d6014d06bb354e589a5cd46d 2024-11-10T03:17:34,140 INFO [Thread-1140 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1140. 2024-11-10T03:17:34,140 DEBUG [Thread-1140 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T03:17:34,140 DEBUG [Thread-1140 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:17:34,140 DEBUG [Thread-1140 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:17:34,140 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T03:17:34,141 INFO [Thread-1146 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1146. 2024-11-10T03:17:34,141 DEBUG [Thread-1146 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T03:17:34,141 DEBUG [Thread-1146 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:17:34,141 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/6a7b8515d6014d06bb354e589a5cd46d, entries=150, sequenceid=538, filesize=12.0 K 2024-11-10T03:17:34,141 DEBUG [Thread-1146 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:17:34,141 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T03:17:34,141 INFO [Thread-1137 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1137. 2024-11-10T03:17:34,141 DEBUG [Thread-1137 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T03:17:34,141 DEBUG [Thread-1137 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:17:34,141 DEBUG [Thread-1137 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:17:34,142 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T03:17:34,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/C/5b015db3fcce437db83f465df162d21d as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/5b015db3fcce437db83f465df162d21d 2024-11-10T03:17:34,143 INFO [Thread-1133 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1133. 2024-11-10T03:17:34,143 DEBUG [Thread-1133 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T03:17:34,143 DEBUG [Thread-1133 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:17:34,143 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T03:17:34,144 DEBUG [Thread-1133 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:17:34,144 INFO [Thread-1148 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1148. 2024-11-10T03:17:34,144 INFO [Thread-1131 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1131. 2024-11-10T03:17:34,144 DEBUG [Thread-1148 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T03:17:34,144 DEBUG [Thread-1148 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:17:34,144 DEBUG [Thread-1131 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T03:17:34,144 DEBUG [Thread-1131 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:17:34,144 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:34,145 DEBUG [Thread-1148 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:17:34,145 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T03:17:34,145 DEBUG [Thread-1131 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:17:34,145 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T03:17:34,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49990 deadline: 1731208714143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:34,146 INFO [Thread-1144 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1144. 2024-11-10T03:17:34,146 DEBUG [Thread-1144 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T03:17:34,146 DEBUG [Thread-1144 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:17:34,146 DEBUG [Thread-1144 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:17:34,146 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T03:17:34,146 INFO [Thread-1142 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1142. 2024-11-10T03:17:34,146 DEBUG [Thread-1142 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T03:17:34,146 DEBUG [Thread-1142 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:17:34,146 DEBUG [Thread-1142 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:17:34,146 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 394d2222f724,38139,1731208572054 for 'test_row_1' in a93817da82260912633bd4e3d359d5b7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T03:17:34,146 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T03:17:34,146 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/5b015db3fcce437db83f465df162d21d, entries=150, sequenceid=538, filesize=12.0 K 2024-11-10T03:17:34,146 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:34,146 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:34,147 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:17:34,147 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for a93817da82260912633bd4e3d359d5b7 in 917ms, sequenceid=538, compaction requested=false 2024-11-10T03:17:34,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for a93817da82260912633bd4e3d359d5b7: 2024-11-10T03:17:34,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:34,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-11-10T03:17:34,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-11-10T03:17:34,150 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-10T03:17:34,150 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3000 sec 2024-11-10T03:17:34,152 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 2.3180 sec 2024-11-10T03:17:36,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-10T03:17:36,003 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T03:17:36,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on a93817da82260912633bd4e3d359d5b7 2024-11-10T03:17:36,004 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a93817da82260912633bd4e3d359d5b7 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-10T03:17:36,004 INFO [Thread-1135 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1135. 2024-11-10T03:17:36,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a93817da82260912633bd4e3d359d5b7, store=A 2024-11-10T03:17:36,004 DEBUG [Thread-1135 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T03:17:36,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:17:36,004 DEBUG [Thread-1135 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:17:36,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a93817da82260912633bd4e3d359d5b7, store=B 2024-11-10T03:17:36,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:17:36,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a93817da82260912633bd4e3d359d5b7, store=C 2024-11-10T03:17:36,004 DEBUG [Thread-1135 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:17:36,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:17:36,004 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T03:17:36,009 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/A/e5735d52fbc44ccb9ad1281256814dfa is 50, key is test_row_0/A:col10/1731208654143/Put/seqid=0 2024-11-10T03:17:36,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742158_1334 (size=12301) 2024-11-10T03:17:36,165 INFO [Thread-1129 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1129. 2024-11-10T03:17:36,165 DEBUG [Thread-1129 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T03:17:36,165 DEBUG [Thread-1129 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:17:36,165 DEBUG [Thread-1129 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:17:36,165 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T03:17:36,165 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-10T03:17:36,165 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 109 2024-11-10T03:17:36,165 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 120 2024-11-10T03:17:36,165 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 74 2024-11-10T03:17:36,165 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 64 2024-11-10T03:17:36,165 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 92 2024-11-10T03:17:36,165 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-10T03:17:36,165 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3006 2024-11-10T03:17:36,165 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2817 2024-11-10T03:17:36,165 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2871 2024-11-10T03:17:36,165 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2962 2024-11-10T03:17:36,165 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2827 2024-11-10T03:17:36,165 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-10T03:17:36,165 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T03:17:36,166 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T03:17:36,166 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:17:36,166 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:17:36,166 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T03:17:36,166 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-10T03:17:36,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-10T03:17:36,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-10T03:17:36,170 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731208656170"}]},"ts":"1731208656170"} 2024-11-10T03:17:36,171 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-10T03:17:36,172 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-10T03:17:36,172 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-10T03:17:36,174 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a93817da82260912633bd4e3d359d5b7, UNASSIGN}] 2024-11-10T03:17:36,175 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a93817da82260912633bd4e3d359d5b7, UNASSIGN 2024-11-10T03:17:36,175 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=a93817da82260912633bd4e3d359d5b7, regionState=CLOSING, regionLocation=394d2222f724,38139,1731208572054 2024-11-10T03:17:36,176 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a93817da82260912633bd4e3d359d5b7, UNASSIGN because future has completed 2024-11-10T03:17:36,177 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-10T03:17:36,177 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; CloseRegionProcedure a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054}] 2024-11-10T03:17:36,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-10T03:17:36,329 INFO [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(122): Close a93817da82260912633bd4e3d359d5b7 2024-11-10T03:17:36,329 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-10T03:17:36,329 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1722): Closing a93817da82260912633bd4e3d359d5b7, disabling compactions & flushes 2024-11-10T03:17:36,330 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:36,415 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=554 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/A/e5735d52fbc44ccb9ad1281256814dfa 2024-11-10T03:17:36,427 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/B/6b1b721ba1ab4c45aaf01595f840ecad is 50, key is test_row_0/B:col10/1731208654143/Put/seqid=0 2024-11-10T03:17:36,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742159_1335 (size=12301) 2024-11-10T03:17:36,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-10T03:17:36,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-10T03:17:36,832 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=554 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/B/6b1b721ba1ab4c45aaf01595f840ecad 2024-11-10T03:17:36,838 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/C/748534d1db8d40ef8aff7d91930c8f04 is 50, key is test_row_0/C:col10/1731208654143/Put/seqid=0 2024-11-10T03:17:36,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742160_1336 (size=12301) 2024-11-10T03:17:37,242 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=554 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/C/748534d1db8d40ef8aff7d91930c8f04 2024-11-10T03:17:37,248 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/A/e5735d52fbc44ccb9ad1281256814dfa as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/e5735d52fbc44ccb9ad1281256814dfa 2024-11-10T03:17:37,254 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/e5735d52fbc44ccb9ad1281256814dfa, entries=150, sequenceid=554, filesize=12.0 K 2024-11-10T03:17:37,255 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/B/6b1b721ba1ab4c45aaf01595f840ecad as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/6b1b721ba1ab4c45aaf01595f840ecad 2024-11-10T03:17:37,260 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/6b1b721ba1ab4c45aaf01595f840ecad, entries=150, sequenceid=554, filesize=12.0 K 2024-11-10T03:17:37,260 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/C/748534d1db8d40ef8aff7d91930c8f04 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/748534d1db8d40ef8aff7d91930c8f04 2024-11-10T03:17:37,266 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/748534d1db8d40ef8aff7d91930c8f04, entries=150, sequenceid=554, filesize=12.0 K 2024-11-10T03:17:37,267 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=6.71 KB/6870 for a93817da82260912633bd4e3d359d5b7 in 1263ms, sequenceid=554, compaction requested=true 2024-11-10T03:17:37,267 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a93817da82260912633bd4e3d359d5b7: 2024-11-10T03:17:37,267 INFO [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:37,267 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:37,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a93817da82260912633bd4e3d359d5b7:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T03:17:37,267 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. after waiting 0 ms 2024-11-10T03:17:37,267 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:37,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:17:37,267 INFO [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(2902): Flushing a93817da82260912633bd4e3d359d5b7 3/3 column families, dataSize=6.71 KB heapSize=20.67 KB 2024-11-10T03:17:37,267 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. because compaction request was cancelled 2024-11-10T03:17:37,267 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. because compaction request was cancelled 2024-11-10T03:17:37,267 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a93817da82260912633bd4e3d359d5b7:A 2024-11-10T03:17:37,267 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a93817da82260912633bd4e3d359d5b7:B 2024-11-10T03:17:37,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a93817da82260912633bd4e3d359d5b7:B, priority=-2147483648, current under compaction store size is 0 2024-11-10T03:17:37,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:17:37,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a93817da82260912633bd4e3d359d5b7:C, priority=-2147483648, current under compaction store size is 1 2024-11-10T03:17:37,267 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. because compaction request was cancelled 2024-11-10T03:17:37,267 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a93817da82260912633bd4e3d359d5b7, store=A 2024-11-10T03:17:37,267 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a93817da82260912633bd4e3d359d5b7:C 2024-11-10T03:17:37,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:17:37,267 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:17:37,267 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a93817da82260912633bd4e3d359d5b7, store=B 2024-11-10T03:17:37,268 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:17:37,268 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a93817da82260912633bd4e3d359d5b7, store=C 2024-11-10T03:17:37,268 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:17:37,272 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/A/5748d94c02294aa1a6db69648e475ab8 is 50, key is test_row_1/A:col10/1731208656163/Put/seqid=0 2024-11-10T03:17:37,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742161_1337 (size=7415) 2024-11-10T03:17:37,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-10T03:17:37,683 INFO [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=558 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/A/5748d94c02294aa1a6db69648e475ab8 2024-11-10T03:17:37,693 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/B/290e2d714c6c4aaab32aa54ca7f49215 is 50, key is test_row_1/B:col10/1731208656163/Put/seqid=0 2024-11-10T03:17:37,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742162_1338 (size=7415) 2024-11-10T03:17:37,733 INFO [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=558 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/B/290e2d714c6c4aaab32aa54ca7f49215 2024-11-10T03:17:37,745 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/C/70919418e84945188c3aa6e5f2cdecac is 50, key is test_row_1/C:col10/1731208656163/Put/seqid=0 2024-11-10T03:17:37,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742163_1339 (size=7415) 2024-11-10T03:17:38,162 INFO [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=558 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/C/70919418e84945188c3aa6e5f2cdecac 2024-11-10T03:17:38,166 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/A/5748d94c02294aa1a6db69648e475ab8 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/5748d94c02294aa1a6db69648e475ab8 2024-11-10T03:17:38,170 INFO [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/5748d94c02294aa1a6db69648e475ab8, entries=50, sequenceid=558, filesize=7.2 K 2024-11-10T03:17:38,170 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/B/290e2d714c6c4aaab32aa54ca7f49215 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/290e2d714c6c4aaab32aa54ca7f49215 2024-11-10T03:17:38,174 INFO [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/290e2d714c6c4aaab32aa54ca7f49215, entries=50, sequenceid=558, filesize=7.2 K 2024-11-10T03:17:38,174 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/.tmp/C/70919418e84945188c3aa6e5f2cdecac as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/70919418e84945188c3aa6e5f2cdecac 2024-11-10T03:17:38,178 INFO [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/70919418e84945188c3aa6e5f2cdecac, entries=50, sequenceid=558, filesize=7.2 K 2024-11-10T03:17:38,178 INFO [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(3140): Finished flush of dataSize ~6.71 KB/6870, heapSize ~20.63 KB/21120, currentSize=0 B/0 for a93817da82260912633bd4e3d359d5b7 in 911ms, sequenceid=558, compaction requested=true 2024-11-10T03:17:38,179 DEBUG [StoreCloser-TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/5d0cd48ca17f445a9c9fb66413b05919, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/8c208cb4ecd946c296ba5ee2b5da7075, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/406dcc221b2240d4a139bbde039adaa1, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/f160c455621c4049b83435a88ab226c3, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/79d58dd597a94b158913873cc60a55ba, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/2c739a58bc5e4997aaf9895d2e0e5110, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/0325c28dd6794bbaaa29dc9378d977cb, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/f5e46cc015074f98b9a17c877e9662c7, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/3c8484b8dab04c36b7a8b2428ee590f5, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/2ccbbf3ae6e04d1994ae03aad09187c5, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/888e5822d88043d1ab27355a7806e830, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/7c745e31543e49af974aaabb46ec6386, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/48f574eceb9b4d3d8e735cb3d27bba2e, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/37a0d8e72a3c4ab1a1bde45715c1233b, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/b6181718ddb349ffb4f4c42fe8052acf, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/4feb3b35091d42c8a86da79b7da7dfe9, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/d4a3ff513c3b4300a5533423600fe6dd, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/58a2785993274f8295c8e084d40e97eb, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/95ba08c1765e41d08fec4d9cf495bcc2, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/284a31358cd14c2fb6f93b91956cb98c, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/29178a79482a41c893f5c24c433d4c5a, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/b74669fda74f4fc5b7d19dc8b5fd88a1, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/12b2750234504d5e9292e2dc3595bb61, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/fd5d50b58eab4acc846c2c56e7588b17, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/738566be993541d6b66e6cc4493f0f5a, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/dea7f65d96254628a297b478c8810980, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/5381ee692eda44d0ab7f456711fa4202, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/25ad4654f6f748a4a4ff026df459b659] to archive 2024-11-10T03:17:38,180 DEBUG [StoreCloser-TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T03:17:38,182 DEBUG [StoreCloser-TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/5d0cd48ca17f445a9c9fb66413b05919 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/5d0cd48ca17f445a9c9fb66413b05919 2024-11-10T03:17:38,183 DEBUG [StoreCloser-TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/8c208cb4ecd946c296ba5ee2b5da7075 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/8c208cb4ecd946c296ba5ee2b5da7075 2024-11-10T03:17:38,184 DEBUG [StoreCloser-TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/406dcc221b2240d4a139bbde039adaa1 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/406dcc221b2240d4a139bbde039adaa1 2024-11-10T03:17:38,185 DEBUG [StoreCloser-TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/f160c455621c4049b83435a88ab226c3 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/f160c455621c4049b83435a88ab226c3 2024-11-10T03:17:38,186 DEBUG [StoreCloser-TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/79d58dd597a94b158913873cc60a55ba to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/79d58dd597a94b158913873cc60a55ba 2024-11-10T03:17:38,187 DEBUG [StoreCloser-TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/2c739a58bc5e4997aaf9895d2e0e5110 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/2c739a58bc5e4997aaf9895d2e0e5110 2024-11-10T03:17:38,188 DEBUG [StoreCloser-TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/0325c28dd6794bbaaa29dc9378d977cb to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/0325c28dd6794bbaaa29dc9378d977cb 2024-11-10T03:17:38,189 DEBUG [StoreCloser-TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/f5e46cc015074f98b9a17c877e9662c7 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/f5e46cc015074f98b9a17c877e9662c7 2024-11-10T03:17:38,190 DEBUG [StoreCloser-TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/3c8484b8dab04c36b7a8b2428ee590f5 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/3c8484b8dab04c36b7a8b2428ee590f5 2024-11-10T03:17:38,191 DEBUG [StoreCloser-TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/2ccbbf3ae6e04d1994ae03aad09187c5 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/2ccbbf3ae6e04d1994ae03aad09187c5 2024-11-10T03:17:38,192 DEBUG [StoreCloser-TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/888e5822d88043d1ab27355a7806e830 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/888e5822d88043d1ab27355a7806e830 2024-11-10T03:17:38,192 DEBUG [StoreCloser-TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/7c745e31543e49af974aaabb46ec6386 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/7c745e31543e49af974aaabb46ec6386 2024-11-10T03:17:38,193 DEBUG [StoreCloser-TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/48f574eceb9b4d3d8e735cb3d27bba2e to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/48f574eceb9b4d3d8e735cb3d27bba2e 2024-11-10T03:17:38,194 DEBUG [StoreCloser-TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/37a0d8e72a3c4ab1a1bde45715c1233b to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/37a0d8e72a3c4ab1a1bde45715c1233b 2024-11-10T03:17:38,195 DEBUG [StoreCloser-TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/b6181718ddb349ffb4f4c42fe8052acf to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/b6181718ddb349ffb4f4c42fe8052acf 2024-11-10T03:17:38,196 DEBUG [StoreCloser-TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/4feb3b35091d42c8a86da79b7da7dfe9 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/4feb3b35091d42c8a86da79b7da7dfe9 2024-11-10T03:17:38,197 DEBUG [StoreCloser-TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/d4a3ff513c3b4300a5533423600fe6dd to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/d4a3ff513c3b4300a5533423600fe6dd 2024-11-10T03:17:38,197 DEBUG [StoreCloser-TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/58a2785993274f8295c8e084d40e97eb to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/58a2785993274f8295c8e084d40e97eb 2024-11-10T03:17:38,198 DEBUG [StoreCloser-TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/95ba08c1765e41d08fec4d9cf495bcc2 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/95ba08c1765e41d08fec4d9cf495bcc2 2024-11-10T03:17:38,199 DEBUG [StoreCloser-TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/284a31358cd14c2fb6f93b91956cb98c to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/284a31358cd14c2fb6f93b91956cb98c 2024-11-10T03:17:38,200 DEBUG [StoreCloser-TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/29178a79482a41c893f5c24c433d4c5a to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/29178a79482a41c893f5c24c433d4c5a 2024-11-10T03:17:38,200 DEBUG [StoreCloser-TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/b74669fda74f4fc5b7d19dc8b5fd88a1 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/b74669fda74f4fc5b7d19dc8b5fd88a1 2024-11-10T03:17:38,201 DEBUG [StoreCloser-TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/12b2750234504d5e9292e2dc3595bb61 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/12b2750234504d5e9292e2dc3595bb61 2024-11-10T03:17:38,202 DEBUG [StoreCloser-TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/fd5d50b58eab4acc846c2c56e7588b17 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/fd5d50b58eab4acc846c2c56e7588b17 2024-11-10T03:17:38,203 DEBUG [StoreCloser-TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/738566be993541d6b66e6cc4493f0f5a to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/738566be993541d6b66e6cc4493f0f5a 2024-11-10T03:17:38,204 DEBUG [StoreCloser-TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/dea7f65d96254628a297b478c8810980 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/dea7f65d96254628a297b478c8810980 2024-11-10T03:17:38,204 DEBUG [StoreCloser-TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/5381ee692eda44d0ab7f456711fa4202 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/5381ee692eda44d0ab7f456711fa4202 2024-11-10T03:17:38,205 DEBUG [StoreCloser-TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/25ad4654f6f748a4a4ff026df459b659 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/25ad4654f6f748a4a4ff026df459b659 2024-11-10T03:17:38,207 DEBUG [StoreCloser-TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/3033361a4f5841fea05de4f7ee541ad3, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/977b9366611f49269f5f2ec72ff1ea7d, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/7bd58034c1164a9a98e73981239ba58a, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/5740a8e4cd344af28d5326f61f530351, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/d4b9f0d0da824cc2baf171c2e3f5177d, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/161433a883984db6964b963faaaa5197, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/1c53949971274fd8aa5001bf5d6b477a, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/e38209c6f193440f80dfd2d897ed9059, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/f72f2a6088c348bc86f9d3d82a314f65, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/424019aec8534f47a56b992fc218405c, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/9e1a0ad7592e485a800e6b8ceb55e570, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/6fa29ec7b85243fabb074283dc9055da, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/0a5687c87bbc43f593c0cd808c353856, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/3112fc8fa9d84f21925a2dfe86f62768, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/26c6a8433dbc47e3be5abbc61534cbc8, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/f2cb4b008b8c43b2b4769855b6cd679d, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/7b3d6903bfed4fbc8808078c2a2a8b8e, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/8f0f041344c34abeb0cbaa906ef503af, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/11f812621f454414b63bdffe82535e9d, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/001d0cdf4edc4ef58b1aabfb190f11a5, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/ec8a659fda5444278b34069211fd5e93, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/25ab8a1351f849398273bb49e0882d65, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/2ad18bfc772749fc9dfcbaa37c280f27, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/f0c3d03b1558419ebcc87e4726174ef1, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/05d723fef028446a83c6a3ca1f064f4b, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/e8a34063b0774fc5b4d7d4a0fbf38532, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/122776a18a3447feb72d73844b534262, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/6eb89f4b4321476e81ca53d98fc25a7d] to archive 2024-11-10T03:17:38,207 DEBUG [StoreCloser-TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T03:17:38,208 DEBUG [StoreCloser-TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/3033361a4f5841fea05de4f7ee541ad3 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/3033361a4f5841fea05de4f7ee541ad3 2024-11-10T03:17:38,209 DEBUG [StoreCloser-TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/977b9366611f49269f5f2ec72ff1ea7d to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/977b9366611f49269f5f2ec72ff1ea7d 2024-11-10T03:17:38,210 DEBUG [StoreCloser-TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/7bd58034c1164a9a98e73981239ba58a to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/7bd58034c1164a9a98e73981239ba58a 2024-11-10T03:17:38,211 DEBUG [StoreCloser-TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/5740a8e4cd344af28d5326f61f530351 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/5740a8e4cd344af28d5326f61f530351 2024-11-10T03:17:38,212 DEBUG [StoreCloser-TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/d4b9f0d0da824cc2baf171c2e3f5177d to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/d4b9f0d0da824cc2baf171c2e3f5177d 2024-11-10T03:17:38,213 DEBUG [StoreCloser-TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/161433a883984db6964b963faaaa5197 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/161433a883984db6964b963faaaa5197 2024-11-10T03:17:38,214 DEBUG [StoreCloser-TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/1c53949971274fd8aa5001bf5d6b477a to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/1c53949971274fd8aa5001bf5d6b477a 2024-11-10T03:17:38,215 DEBUG [StoreCloser-TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/e38209c6f193440f80dfd2d897ed9059 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/e38209c6f193440f80dfd2d897ed9059 2024-11-10T03:17:38,216 DEBUG [StoreCloser-TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/f72f2a6088c348bc86f9d3d82a314f65 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/f72f2a6088c348bc86f9d3d82a314f65 2024-11-10T03:17:38,217 DEBUG [StoreCloser-TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/424019aec8534f47a56b992fc218405c to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/424019aec8534f47a56b992fc218405c 2024-11-10T03:17:38,218 DEBUG [StoreCloser-TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/9e1a0ad7592e485a800e6b8ceb55e570 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/9e1a0ad7592e485a800e6b8ceb55e570 2024-11-10T03:17:38,219 DEBUG [StoreCloser-TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/6fa29ec7b85243fabb074283dc9055da to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/6fa29ec7b85243fabb074283dc9055da 2024-11-10T03:17:38,220 DEBUG [StoreCloser-TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/0a5687c87bbc43f593c0cd808c353856 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/0a5687c87bbc43f593c0cd808c353856 2024-11-10T03:17:38,223 DEBUG [StoreCloser-TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/3112fc8fa9d84f21925a2dfe86f62768 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/3112fc8fa9d84f21925a2dfe86f62768 2024-11-10T03:17:38,224 DEBUG [StoreCloser-TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/26c6a8433dbc47e3be5abbc61534cbc8 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/26c6a8433dbc47e3be5abbc61534cbc8 2024-11-10T03:17:38,225 DEBUG [StoreCloser-TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/f2cb4b008b8c43b2b4769855b6cd679d to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/f2cb4b008b8c43b2b4769855b6cd679d 2024-11-10T03:17:38,226 DEBUG [StoreCloser-TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/7b3d6903bfed4fbc8808078c2a2a8b8e to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/7b3d6903bfed4fbc8808078c2a2a8b8e 2024-11-10T03:17:38,227 DEBUG [StoreCloser-TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/8f0f041344c34abeb0cbaa906ef503af to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/8f0f041344c34abeb0cbaa906ef503af 2024-11-10T03:17:38,227 DEBUG [StoreCloser-TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/11f812621f454414b63bdffe82535e9d to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/11f812621f454414b63bdffe82535e9d 2024-11-10T03:17:38,228 DEBUG [StoreCloser-TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/001d0cdf4edc4ef58b1aabfb190f11a5 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/001d0cdf4edc4ef58b1aabfb190f11a5 2024-11-10T03:17:38,229 DEBUG [StoreCloser-TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/ec8a659fda5444278b34069211fd5e93 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/ec8a659fda5444278b34069211fd5e93 2024-11-10T03:17:38,230 DEBUG [StoreCloser-TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/25ab8a1351f849398273bb49e0882d65 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/25ab8a1351f849398273bb49e0882d65 2024-11-10T03:17:38,231 DEBUG [StoreCloser-TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/2ad18bfc772749fc9dfcbaa37c280f27 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/2ad18bfc772749fc9dfcbaa37c280f27 2024-11-10T03:17:38,232 DEBUG [StoreCloser-TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/f0c3d03b1558419ebcc87e4726174ef1 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/f0c3d03b1558419ebcc87e4726174ef1 2024-11-10T03:17:38,232 DEBUG [StoreCloser-TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/05d723fef028446a83c6a3ca1f064f4b to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/05d723fef028446a83c6a3ca1f064f4b 2024-11-10T03:17:38,233 DEBUG [StoreCloser-TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/e8a34063b0774fc5b4d7d4a0fbf38532 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/e8a34063b0774fc5b4d7d4a0fbf38532 2024-11-10T03:17:38,234 DEBUG [StoreCloser-TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/122776a18a3447feb72d73844b534262 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/122776a18a3447feb72d73844b534262 2024-11-10T03:17:38,235 DEBUG [StoreCloser-TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/6eb89f4b4321476e81ca53d98fc25a7d to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/6eb89f4b4321476e81ca53d98fc25a7d 2024-11-10T03:17:38,236 DEBUG [StoreCloser-TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/da2f8bdc98d44f989ebe19e1835f3858, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/5dc01dcc13a34c879adabaf80c5e2fe7, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/20ff6cff7e744e5d8e83553111e4005c, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/c5c7724d0de94fcf98acabc47b2440e5, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/cd5d734f48fa4d4aad7563ef742b0d43, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/1a1a95d7de704a0f9e2f73d9fd803ff3, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/2adfe9d89dfb4b02ad904ef6c0c3105a, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/209b1080fa7b4727bf9a52265ba790e0, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/12792fdd471d400686a67d9b71b85a60, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/7c8240e4279640d1a979a9d8cc4a3ff3, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/fbc8ecf0bbbc42c7a8ec5472e99397b8, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/83d8b1ff5a2a4a49b0c3a9201f7ea446, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/13814e730b8443e09ddd1946a55406df, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/0e1701de081640d08b091a10a42805f8, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/8e6259fdb1df4cf99e5fd6801c7b1c4c, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/87f9a104a38041dc9c1a25c92a18c696, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/78598c9ae8dd487dacf716cad4a34c56, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/a6a7335e58b541aeaf5063373ed6d989, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/4241d7d441324330b20dc0395d97ccbc, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/6d98934f4c424e8383b335f13bf37384, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/403e954d5ba049beb30576a4926d3ea6, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/dd5db3dc21504538b16f24558a94f7c2, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/43dc48163c1c4ca3bdb0150bf5db3220, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/9d716c96e64d4439958d4cf417ce2581, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/0d09f75fa7b4436daf3dcc9a59112396, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/34d96f46a1c041cab11dbaae559077fe, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/2bab44195b0148198f3a2b005bb23384, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/e9778bb4684547279efa6a5f8decef1c] to archive 2024-11-10T03:17:38,237 DEBUG [StoreCloser-TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T03:17:38,238 DEBUG [StoreCloser-TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/da2f8bdc98d44f989ebe19e1835f3858 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/da2f8bdc98d44f989ebe19e1835f3858 2024-11-10T03:17:38,239 DEBUG [StoreCloser-TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/5dc01dcc13a34c879adabaf80c5e2fe7 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/5dc01dcc13a34c879adabaf80c5e2fe7 2024-11-10T03:17:38,240 DEBUG [StoreCloser-TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/20ff6cff7e744e5d8e83553111e4005c to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/20ff6cff7e744e5d8e83553111e4005c 2024-11-10T03:17:38,241 DEBUG [StoreCloser-TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/c5c7724d0de94fcf98acabc47b2440e5 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/c5c7724d0de94fcf98acabc47b2440e5 2024-11-10T03:17:38,248 DEBUG [StoreCloser-TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/cd5d734f48fa4d4aad7563ef742b0d43 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/cd5d734f48fa4d4aad7563ef742b0d43 2024-11-10T03:17:38,250 DEBUG [StoreCloser-TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/1a1a95d7de704a0f9e2f73d9fd803ff3 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/1a1a95d7de704a0f9e2f73d9fd803ff3 2024-11-10T03:17:38,250 DEBUG [StoreCloser-TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/2adfe9d89dfb4b02ad904ef6c0c3105a to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/2adfe9d89dfb4b02ad904ef6c0c3105a 2024-11-10T03:17:38,251 DEBUG [StoreCloser-TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/209b1080fa7b4727bf9a52265ba790e0 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/209b1080fa7b4727bf9a52265ba790e0 2024-11-10T03:17:38,252 DEBUG [StoreCloser-TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/12792fdd471d400686a67d9b71b85a60 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/12792fdd471d400686a67d9b71b85a60 2024-11-10T03:17:38,253 DEBUG [StoreCloser-TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/7c8240e4279640d1a979a9d8cc4a3ff3 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/7c8240e4279640d1a979a9d8cc4a3ff3 2024-11-10T03:17:38,254 DEBUG [StoreCloser-TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/fbc8ecf0bbbc42c7a8ec5472e99397b8 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/fbc8ecf0bbbc42c7a8ec5472e99397b8 2024-11-10T03:17:38,254 DEBUG [StoreCloser-TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/83d8b1ff5a2a4a49b0c3a9201f7ea446 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/83d8b1ff5a2a4a49b0c3a9201f7ea446 2024-11-10T03:17:38,255 DEBUG [StoreCloser-TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/13814e730b8443e09ddd1946a55406df to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/13814e730b8443e09ddd1946a55406df 2024-11-10T03:17:38,256 DEBUG [StoreCloser-TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/0e1701de081640d08b091a10a42805f8 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/0e1701de081640d08b091a10a42805f8 2024-11-10T03:17:38,257 DEBUG [StoreCloser-TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/8e6259fdb1df4cf99e5fd6801c7b1c4c to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/8e6259fdb1df4cf99e5fd6801c7b1c4c 2024-11-10T03:17:38,258 DEBUG [StoreCloser-TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/87f9a104a38041dc9c1a25c92a18c696 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/87f9a104a38041dc9c1a25c92a18c696 2024-11-10T03:17:38,259 DEBUG [StoreCloser-TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/78598c9ae8dd487dacf716cad4a34c56 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/78598c9ae8dd487dacf716cad4a34c56 2024-11-10T03:17:38,260 DEBUG [StoreCloser-TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/a6a7335e58b541aeaf5063373ed6d989 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/a6a7335e58b541aeaf5063373ed6d989 2024-11-10T03:17:38,261 DEBUG [StoreCloser-TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/4241d7d441324330b20dc0395d97ccbc to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/4241d7d441324330b20dc0395d97ccbc 2024-11-10T03:17:38,262 DEBUG [StoreCloser-TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/6d98934f4c424e8383b335f13bf37384 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/6d98934f4c424e8383b335f13bf37384 2024-11-10T03:17:38,263 DEBUG [StoreCloser-TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/403e954d5ba049beb30576a4926d3ea6 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/403e954d5ba049beb30576a4926d3ea6 2024-11-10T03:17:38,264 DEBUG [StoreCloser-TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/dd5db3dc21504538b16f24558a94f7c2 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/dd5db3dc21504538b16f24558a94f7c2 2024-11-10T03:17:38,265 DEBUG [StoreCloser-TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/43dc48163c1c4ca3bdb0150bf5db3220 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/43dc48163c1c4ca3bdb0150bf5db3220 2024-11-10T03:17:38,266 DEBUG [StoreCloser-TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/9d716c96e64d4439958d4cf417ce2581 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/9d716c96e64d4439958d4cf417ce2581 2024-11-10T03:17:38,266 DEBUG [StoreCloser-TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/0d09f75fa7b4436daf3dcc9a59112396 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/0d09f75fa7b4436daf3dcc9a59112396 2024-11-10T03:17:38,267 DEBUG [StoreCloser-TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/34d96f46a1c041cab11dbaae559077fe to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/34d96f46a1c041cab11dbaae559077fe 2024-11-10T03:17:38,268 DEBUG [StoreCloser-TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/2bab44195b0148198f3a2b005bb23384 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/2bab44195b0148198f3a2b005bb23384 2024-11-10T03:17:38,269 DEBUG [StoreCloser-TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/e9778bb4684547279efa6a5f8decef1c to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/e9778bb4684547279efa6a5f8decef1c 2024-11-10T03:17:38,274 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/recovered.edits/561.seqid, newMaxSeqId=561, maxSeqId=1 2024-11-10T03:17:38,274 INFO [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. 2024-11-10T03:17:38,274 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1676): Region close journal for a93817da82260912633bd4e3d359d5b7: Waiting for close lock at 1731208656329Running coprocessor pre-close hooks at 1731208656329Disabling compacts and flushes for region at 1731208656329Disabling writes for close at 1731208657267 (+938 ms)Obtaining lock to block concurrent updates at 1731208657267Preparing flush snapshotting stores in a93817da82260912633bd4e3d359d5b7 at 1731208657267Finished memstore snapshotting TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7., syncing WAL and waiting on mvcc, flushsize=dataSize=6870, getHeapSize=21120, getOffHeapSize=0, getCellsCount=150 at 1731208657268 (+1 ms)Flushing stores of TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7. at 1731208657268Flushing a93817da82260912633bd4e3d359d5b7/A: creating writer at 1731208657268Flushing a93817da82260912633bd4e3d359d5b7/A: appending metadata at 1731208657271 (+3 ms)Flushing a93817da82260912633bd4e3d359d5b7/A: closing flushed file at 1731208657271Flushing a93817da82260912633bd4e3d359d5b7/B: creating writer at 1731208657689 (+418 ms)Flushing a93817da82260912633bd4e3d359d5b7/B: appending metadata at 1731208657692 (+3 ms)Flushing a93817da82260912633bd4e3d359d5b7/B: closing flushed file at 1731208657692Flushing a93817da82260912633bd4e3d359d5b7/C: creating writer at 1731208657741 (+49 ms)Flushing a93817da82260912633bd4e3d359d5b7/C: appending metadata at 1731208657744 (+3 ms)Flushing a93817da82260912633bd4e3d359d5b7/C: closing flushed file at 1731208657744Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@29571585: reopening flushed file at 1731208658166 (+422 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2db73584: reopening flushed file at 1731208658170 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@51c9a76e: reopening flushed file at 1731208658174 (+4 ms)Finished flush of dataSize ~6.71 KB/6870, heapSize ~20.63 KB/21120, currentSize=0 B/0 for a93817da82260912633bd4e3d359d5b7 in 911ms, sequenceid=558, compaction requested=true at 1731208658178 (+4 ms)Writing region close event to WAL at 1731208658271 (+93 ms)Running coprocessor post-close hooks at 1731208658274 (+3 ms)Closed at 1731208658274 2024-11-10T03:17:38,276 INFO [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(157): Closed a93817da82260912633bd4e3d359d5b7 2024-11-10T03:17:38,277 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=a93817da82260912633bd4e3d359d5b7, regionState=CLOSED 2024-11-10T03:17:38,278 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=85, ppid=84, state=RUNNABLE, hasLock=false; CloseRegionProcedure a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 because future has completed 2024-11-10T03:17:38,280 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-10T03:17:38,280 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; CloseRegionProcedure a93817da82260912633bd4e3d359d5b7, server=394d2222f724,38139,1731208572054 in 2.1020 sec 2024-11-10T03:17:38,281 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=84, resume processing ppid=83 2024-11-10T03:17:38,281 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, ppid=83, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a93817da82260912633bd4e3d359d5b7, UNASSIGN in 2.1060 sec 2024-11-10T03:17:38,283 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-10T03:17:38,283 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.1100 sec 2024-11-10T03:17:38,284 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731208658284"}]},"ts":"1731208658284"} 2024-11-10T03:17:38,285 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-10T03:17:38,285 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-10T03:17:38,286 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.1180 sec 2024-11-10T03:17:38,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-10T03:17:38,323 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-10T03:17:38,323 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-10T03:17:38,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T03:17:38,325 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=86, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T03:17:38,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-10T03:17:38,325 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=86, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T03:17:38,328 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7 2024-11-10T03:17:38,330 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A, FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B, FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C, FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/recovered.edits] 2024-11-10T03:17:38,332 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/1939ba45854e436aac96c886293538bd to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/1939ba45854e436aac96c886293538bd 2024-11-10T03:17:38,333 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/5748d94c02294aa1a6db69648e475ab8 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/5748d94c02294aa1a6db69648e475ab8 2024-11-10T03:17:38,334 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/8e401fbd1fb4479083f8583607e807fa to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/8e401fbd1fb4479083f8583607e807fa 2024-11-10T03:17:38,335 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/e5735d52fbc44ccb9ad1281256814dfa to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/A/e5735d52fbc44ccb9ad1281256814dfa 2024-11-10T03:17:38,337 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/290e2d714c6c4aaab32aa54ca7f49215 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/290e2d714c6c4aaab32aa54ca7f49215 2024-11-10T03:17:38,338 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/6a7b8515d6014d06bb354e589a5cd46d to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/6a7b8515d6014d06bb354e589a5cd46d 2024-11-10T03:17:38,339 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/6b1b721ba1ab4c45aaf01595f840ecad to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/6b1b721ba1ab4c45aaf01595f840ecad 2024-11-10T03:17:38,339 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/edb61756398d4ebe8622960d7dc4f0ac to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/B/edb61756398d4ebe8622960d7dc4f0ac 2024-11-10T03:17:38,341 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/5b015db3fcce437db83f465df162d21d to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/5b015db3fcce437db83f465df162d21d 2024-11-10T03:17:38,341 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/70919418e84945188c3aa6e5f2cdecac to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/70919418e84945188c3aa6e5f2cdecac 2024-11-10T03:17:38,342 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/748534d1db8d40ef8aff7d91930c8f04 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/748534d1db8d40ef8aff7d91930c8f04 2024-11-10T03:17:38,343 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/8fc3599aedfc4ede9230e4fc55a92628 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/C/8fc3599aedfc4ede9230e4fc55a92628 2024-11-10T03:17:38,345 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/recovered.edits/561.seqid to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7/recovered.edits/561.seqid 2024-11-10T03:17:38,345 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/a93817da82260912633bd4e3d359d5b7 2024-11-10T03:17:38,345 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-10T03:17:38,347 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=86, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T03:17:38,350 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-10T03:17:38,352 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-10T03:17:38,352 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=86, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T03:17:38,353 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-10T03:17:38,353 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731208658353"}]},"ts":"9223372036854775807"} 2024-11-10T03:17:38,354 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-10T03:17:38,354 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => a93817da82260912633bd4e3d359d5b7, NAME => 'TestAcidGuarantees,,1731208631918.a93817da82260912633bd4e3d359d5b7.', STARTKEY => '', ENDKEY => ''}] 2024-11-10T03:17:38,354 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-10T03:17:38,354 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731208658354"}]},"ts":"9223372036854775807"} 2024-11-10T03:17:38,355 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-10T03:17:38,356 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=86, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T03:17:38,357 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 33 msec 2024-11-10T03:17:38,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-10T03:17:38,433 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-10T03:17:38,434 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-10T03:17:38,478 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=250 (was 248) Potentially hanging thread: HFileArchiver-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1264325177_22 at /127.0.0.1:39308 [Waiting for operation #313] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-972336514_22 at /127.0.0.1:60290 [Waiting for operation #288] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=455 (was 455), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=690 (was 754), ProcessCount=11 (was 11), AvailableMemoryMB=1009 (was 1647) 2024-11-10T03:17:38,487 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=250, OpenFileDescriptor=455, MaxFileDescriptor=1048576, SystemLoadAverage=690, ProcessCount=11, AvailableMemoryMB=1009 2024-11-10T03:17:38,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-10T03:17:38,489 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T03:17:38,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] procedure2.ProcedureExecutor(1139): Stored pid=87, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-10T03:17:38,491 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-10T03:17:38,492 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:38,492 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 87 2024-11-10T03:17:38,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-10T03:17:38,493 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-10T03:17:38,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742164_1340 (size=1061) 2024-11-10T03:17:38,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-10T03:17:38,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-10T03:17:38,902 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8 2024-11-10T03:17:38,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742165_1341 (size=53) 2024-11-10T03:17:39,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-10T03:17:39,311 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T03:17:39,311 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing c6879c5307a3686e6731f46d96d3ca08, disabling compactions & flushes 2024-11-10T03:17:39,311 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:39,311 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:39,311 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. after waiting 0 ms 2024-11-10T03:17:39,311 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:39,311 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:39,311 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for c6879c5307a3686e6731f46d96d3ca08: Waiting for close lock at 1731208659311Disabling compacts and flushes for region at 1731208659311Disabling writes for close at 1731208659311Writing region close event to WAL at 1731208659311Closed at 1731208659311 2024-11-10T03:17:39,313 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-10T03:17:39,314 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731208659313"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731208659313"}]},"ts":"1731208659313"} 2024-11-10T03:17:39,316 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-10T03:17:39,317 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-10T03:17:39,317 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731208659317"}]},"ts":"1731208659317"} 2024-11-10T03:17:39,319 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-10T03:17:39,319 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c6879c5307a3686e6731f46d96d3ca08, ASSIGN}] 2024-11-10T03:17:39,320 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c6879c5307a3686e6731f46d96d3ca08, ASSIGN 2024-11-10T03:17:39,321 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c6879c5307a3686e6731f46d96d3ca08, ASSIGN; state=OFFLINE, location=394d2222f724,38139,1731208572054; forceNewPlan=false, retain=false 2024-11-10T03:17:39,472 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=c6879c5307a3686e6731f46d96d3ca08, regionState=OPENING, regionLocation=394d2222f724,38139,1731208572054 2024-11-10T03:17:39,474 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c6879c5307a3686e6731f46d96d3ca08, ASSIGN because future has completed 2024-11-10T03:17:39,474 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE, hasLock=false; OpenRegionProcedure c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054}] 2024-11-10T03:17:39,628 INFO [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:39,629 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(7752): Opening region: {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} 2024-11-10T03:17:39,629 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees c6879c5307a3686e6731f46d96d3ca08 2024-11-10T03:17:39,629 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T03:17:39,629 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(7794): checking encryption for c6879c5307a3686e6731f46d96d3ca08 2024-11-10T03:17:39,629 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(7797): checking classloading for c6879c5307a3686e6731f46d96d3ca08 2024-11-10T03:17:39,630 INFO [StoreOpener-c6879c5307a3686e6731f46d96d3ca08-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region c6879c5307a3686e6731f46d96d3ca08 2024-11-10T03:17:39,631 INFO [StoreOpener-c6879c5307a3686e6731f46d96d3ca08-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T03:17:39,632 INFO [StoreOpener-c6879c5307a3686e6731f46d96d3ca08-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c6879c5307a3686e6731f46d96d3ca08 columnFamilyName A 2024-11-10T03:17:39,632 DEBUG [StoreOpener-c6879c5307a3686e6731f46d96d3ca08-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:39,632 INFO [StoreOpener-c6879c5307a3686e6731f46d96d3ca08-1 {}] regionserver.HStore(327): Store=c6879c5307a3686e6731f46d96d3ca08/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T03:17:39,632 INFO [StoreOpener-c6879c5307a3686e6731f46d96d3ca08-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region c6879c5307a3686e6731f46d96d3ca08 2024-11-10T03:17:39,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-10T03:17:39,633 INFO [StoreOpener-c6879c5307a3686e6731f46d96d3ca08-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T03:17:39,633 INFO [StoreOpener-c6879c5307a3686e6731f46d96d3ca08-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c6879c5307a3686e6731f46d96d3ca08 columnFamilyName B 2024-11-10T03:17:39,633 DEBUG [StoreOpener-c6879c5307a3686e6731f46d96d3ca08-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:39,634 INFO [StoreOpener-c6879c5307a3686e6731f46d96d3ca08-1 {}] regionserver.HStore(327): Store=c6879c5307a3686e6731f46d96d3ca08/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T03:17:39,634 INFO [StoreOpener-c6879c5307a3686e6731f46d96d3ca08-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region c6879c5307a3686e6731f46d96d3ca08 2024-11-10T03:17:39,635 INFO [StoreOpener-c6879c5307a3686e6731f46d96d3ca08-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T03:17:39,635 INFO [StoreOpener-c6879c5307a3686e6731f46d96d3ca08-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c6879c5307a3686e6731f46d96d3ca08 columnFamilyName C 2024-11-10T03:17:39,635 DEBUG [StoreOpener-c6879c5307a3686e6731f46d96d3ca08-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:39,635 INFO [StoreOpener-c6879c5307a3686e6731f46d96d3ca08-1 {}] regionserver.HStore(327): Store=c6879c5307a3686e6731f46d96d3ca08/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T03:17:39,636 INFO [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:39,636 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1038): replaying wal for c6879c5307a3686e6731f46d96d3ca08 2024-11-10T03:17:39,636 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08 2024-11-10T03:17:39,636 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08 2024-11-10T03:17:39,637 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1048): stopping wal replay for c6879c5307a3686e6731f46d96d3ca08 2024-11-10T03:17:39,637 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1060): Cleaning up temporary data for c6879c5307a3686e6731f46d96d3ca08 2024-11-10T03:17:39,638 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T03:17:39,638 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1093): writing seq id for c6879c5307a3686e6731f46d96d3ca08 2024-11-10T03:17:39,640 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T03:17:39,641 INFO [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1114): Opened c6879c5307a3686e6731f46d96d3ca08; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58996628, jitterRate=-0.12088173627853394}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T03:17:39,641 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c6879c5307a3686e6731f46d96d3ca08 2024-11-10T03:17:39,641 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1006): Region open journal for c6879c5307a3686e6731f46d96d3ca08: Running coprocessor pre-open hook at 1731208659629Writing region info on filesystem at 1731208659629Initializing all the Stores at 1731208659630 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731208659630Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731208659630Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731208659630Cleaning up temporary data from old regions at 1731208659637 (+7 ms)Running coprocessor post-open hooks at 1731208659641 (+4 ms)Region opened successfully at 1731208659641 2024-11-10T03:17:39,642 INFO [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., pid=89, masterSystemTime=1731208659626 2024-11-10T03:17:39,644 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:39,644 INFO [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:39,644 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=c6879c5307a3686e6731f46d96d3ca08, regionState=OPEN, openSeqNum=2, regionLocation=394d2222f724,38139,1731208572054 2024-11-10T03:17:39,646 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=89, ppid=88, state=RUNNABLE, hasLock=false; OpenRegionProcedure c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 because future has completed 2024-11-10T03:17:39,648 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-10T03:17:39,648 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; OpenRegionProcedure c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 in 173 msec 2024-11-10T03:17:39,650 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=87 2024-11-10T03:17:39,650 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=87, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c6879c5307a3686e6731f46d96d3ca08, ASSIGN in 329 msec 2024-11-10T03:17:39,651 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-10T03:17:39,651 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731208659651"}]},"ts":"1731208659651"} 2024-11-10T03:17:39,652 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-10T03:17:39,653 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-10T03:17:39,654 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1640 sec 2024-11-10T03:17:40,311 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-10T03:17:40,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-10T03:17:40,644 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-10T03:17:40,645 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@99b61e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:17:40,645 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 394d2222f724,37299,-1 for getting cluster id 2024-11-10T03:17:40,645 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T03:17:40,646 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f5c6727-65f7-4659-afe8-8e9a55664c78' 2024-11-10T03:17:40,646 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T03:17:40,647 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f5c6727-65f7-4659-afe8-8e9a55664c78" 2024-11-10T03:17:40,647 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31fe1017, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:17:40,647 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [394d2222f724,37299,-1] 2024-11-10T03:17:40,647 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T03:17:40,647 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:17:40,648 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40362, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T03:17:40,649 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69fbead2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:17:40,649 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T03:17:40,650 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=394d2222f724,38139,1731208572054, seqNum=-1] 2024-11-10T03:17:40,650 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T03:17:40,651 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53912, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T03:17:40,651 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-10T03:17:40,652 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 394d2222f724,37299,1731208571229 2024-11-10T03:17:40,652 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@659a1f12 2024-11-10T03:17:40,652 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-10T03:17:40,653 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40370, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-10T03:17:40,655 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-10T03:17:40,655 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T03:17:40,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] procedure2.ProcedureExecutor(1139): Stored pid=90, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-10T03:17:40,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742166_1342 (size=1097) 2024-11-10T03:17:40,667 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-10T03:17:40,667 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-10T03:17:40,670 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-10T03:17:40,673 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c6879c5307a3686e6731f46d96d3ca08, REOPEN/MOVE}] 2024-11-10T03:17:40,673 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c6879c5307a3686e6731f46d96d3ca08, REOPEN/MOVE 2024-11-10T03:17:40,674 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=c6879c5307a3686e6731f46d96d3ca08, regionState=CLOSING, regionLocation=394d2222f724,38139,1731208572054 2024-11-10T03:17:40,675 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c6879c5307a3686e6731f46d96d3ca08, REOPEN/MOVE because future has completed 2024-11-10T03:17:40,675 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-10T03:17:40,676 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=93, ppid=92, state=RUNNABLE, hasLock=false; CloseRegionProcedure c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054}] 2024-11-10T03:17:40,828 INFO [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(122): Close c6879c5307a3686e6731f46d96d3ca08 2024-11-10T03:17:40,828 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-10T03:17:40,828 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1722): Closing c6879c5307a3686e6731f46d96d3ca08, disabling compactions & flushes 2024-11-10T03:17:40,828 INFO [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:40,828 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:40,828 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. after waiting 0 ms 2024-11-10T03:17:40,828 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:40,831 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-10T03:17:40,832 INFO [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:40,832 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1676): Region close journal for c6879c5307a3686e6731f46d96d3ca08: Waiting for close lock at 1731208660828Running coprocessor pre-close hooks at 1731208660828Disabling compacts and flushes for region at 1731208660828Disabling writes for close at 1731208660828Writing region close event to WAL at 1731208660828Running coprocessor post-close hooks at 1731208660831 (+3 ms)Closed at 1731208660832 (+1 ms) 2024-11-10T03:17:40,832 WARN [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegionServer(3299): Not adding moved region record: c6879c5307a3686e6731f46d96d3ca08 to self. 2024-11-10T03:17:40,833 INFO [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(157): Closed c6879c5307a3686e6731f46d96d3ca08 2024-11-10T03:17:40,834 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=c6879c5307a3686e6731f46d96d3ca08, regionState=CLOSED 2024-11-10T03:17:40,835 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=93, ppid=92, state=RUNNABLE, hasLock=false; CloseRegionProcedure c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 because future has completed 2024-11-10T03:17:40,838 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=93, resume processing ppid=92 2024-11-10T03:17:40,838 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, ppid=92, state=SUCCESS, hasLock=false; CloseRegionProcedure c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 in 160 msec 2024-11-10T03:17:40,838 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c6879c5307a3686e6731f46d96d3ca08, REOPEN/MOVE; state=CLOSED, location=394d2222f724,38139,1731208572054; forceNewPlan=false, retain=true 2024-11-10T03:17:40,989 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=c6879c5307a3686e6731f46d96d3ca08, regionState=OPENING, regionLocation=394d2222f724,38139,1731208572054 2024-11-10T03:17:40,992 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c6879c5307a3686e6731f46d96d3ca08, REOPEN/MOVE because future has completed 2024-11-10T03:17:40,993 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=92, state=RUNNABLE, hasLock=false; OpenRegionProcedure c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054}] 2024-11-10T03:17:41,149 INFO [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:41,150 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(7752): Opening region: {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} 2024-11-10T03:17:41,150 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees c6879c5307a3686e6731f46d96d3ca08 2024-11-10T03:17:41,150 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T03:17:41,150 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(7794): checking encryption for c6879c5307a3686e6731f46d96d3ca08 2024-11-10T03:17:41,150 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(7797): checking classloading for c6879c5307a3686e6731f46d96d3ca08 2024-11-10T03:17:41,152 INFO [StoreOpener-c6879c5307a3686e6731f46d96d3ca08-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region c6879c5307a3686e6731f46d96d3ca08 2024-11-10T03:17:41,153 INFO [StoreOpener-c6879c5307a3686e6731f46d96d3ca08-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T03:17:41,154 INFO [StoreOpener-c6879c5307a3686e6731f46d96d3ca08-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c6879c5307a3686e6731f46d96d3ca08 columnFamilyName A 2024-11-10T03:17:41,155 DEBUG [StoreOpener-c6879c5307a3686e6731f46d96d3ca08-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:41,156 INFO [StoreOpener-c6879c5307a3686e6731f46d96d3ca08-1 {}] regionserver.HStore(327): Store=c6879c5307a3686e6731f46d96d3ca08/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T03:17:41,157 INFO [StoreOpener-c6879c5307a3686e6731f46d96d3ca08-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region c6879c5307a3686e6731f46d96d3ca08 2024-11-10T03:17:41,158 INFO [StoreOpener-c6879c5307a3686e6731f46d96d3ca08-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T03:17:41,158 INFO [StoreOpener-c6879c5307a3686e6731f46d96d3ca08-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c6879c5307a3686e6731f46d96d3ca08 columnFamilyName B 2024-11-10T03:17:41,158 DEBUG [StoreOpener-c6879c5307a3686e6731f46d96d3ca08-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:41,159 INFO [StoreOpener-c6879c5307a3686e6731f46d96d3ca08-1 {}] regionserver.HStore(327): Store=c6879c5307a3686e6731f46d96d3ca08/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T03:17:41,159 INFO [StoreOpener-c6879c5307a3686e6731f46d96d3ca08-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region c6879c5307a3686e6731f46d96d3ca08 2024-11-10T03:17:41,159 INFO [StoreOpener-c6879c5307a3686e6731f46d96d3ca08-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T03:17:41,160 INFO [StoreOpener-c6879c5307a3686e6731f46d96d3ca08-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c6879c5307a3686e6731f46d96d3ca08 columnFamilyName C 2024-11-10T03:17:41,160 DEBUG [StoreOpener-c6879c5307a3686e6731f46d96d3ca08-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:41,160 INFO [StoreOpener-c6879c5307a3686e6731f46d96d3ca08-1 {}] regionserver.HStore(327): Store=c6879c5307a3686e6731f46d96d3ca08/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T03:17:41,160 INFO [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:41,160 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1038): replaying wal for c6879c5307a3686e6731f46d96d3ca08 2024-11-10T03:17:41,161 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08 2024-11-10T03:17:41,162 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08 2024-11-10T03:17:41,164 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1048): stopping wal replay for c6879c5307a3686e6731f46d96d3ca08 2024-11-10T03:17:41,164 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1060): Cleaning up temporary data for c6879c5307a3686e6731f46d96d3ca08 2024-11-10T03:17:41,164 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T03:17:41,166 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1093): writing seq id for c6879c5307a3686e6731f46d96d3ca08 2024-11-10T03:17:41,167 INFO [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1114): Opened c6879c5307a3686e6731f46d96d3ca08; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64967288, jitterRate=-0.03191196918487549}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T03:17:41,167 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c6879c5307a3686e6731f46d96d3ca08 2024-11-10T03:17:41,168 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1006): Region open journal for c6879c5307a3686e6731f46d96d3ca08: Running coprocessor pre-open hook at 1731208661150Writing region info on filesystem at 1731208661150Initializing all the Stores at 1731208661151 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731208661151Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731208661152 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731208661152Cleaning up temporary data from old regions at 1731208661164 (+12 ms)Running coprocessor post-open hooks at 1731208661167 (+3 ms)Region opened successfully at 1731208661168 (+1 ms) 2024-11-10T03:17:41,169 INFO [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., pid=94, masterSystemTime=1731208661146 2024-11-10T03:17:41,171 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:41,171 INFO [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:41,172 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=c6879c5307a3686e6731f46d96d3ca08, regionState=OPEN, openSeqNum=5, regionLocation=394d2222f724,38139,1731208572054 2024-11-10T03:17:41,174 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=92, state=RUNNABLE, hasLock=false; OpenRegionProcedure c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 because future has completed 2024-11-10T03:17:41,178 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=92 2024-11-10T03:17:41,178 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=92, state=SUCCESS, hasLock=false; OpenRegionProcedure c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 in 182 msec 2024-11-10T03:17:41,181 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-11-10T03:17:41,181 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c6879c5307a3686e6731f46d96d3ca08, REOPEN/MOVE in 506 msec 2024-11-10T03:17:41,183 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-11-10T03:17:41,183 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 511 msec 2024-11-10T03:17:41,185 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 529 msec 2024-11-10T03:17:41,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-10T03:17:41,186 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-10T03:17:41,186 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a1929e6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:17:41,186 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 394d2222f724,37299,-1 for getting cluster id 2024-11-10T03:17:41,186 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T03:17:41,187 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f5c6727-65f7-4659-afe8-8e9a55664c78' 2024-11-10T03:17:41,187 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T03:17:41,187 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f5c6727-65f7-4659-afe8-8e9a55664c78" 2024-11-10T03:17:41,188 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1da1c474, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:17:41,188 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [394d2222f724,37299,-1] 2024-11-10T03:17:41,188 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T03:17:41,188 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:17:41,189 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40378, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T03:17:41,189 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d84e5e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:17:41,190 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b673756, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:17:41,190 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 394d2222f724,37299,-1 for getting cluster id 2024-11-10T03:17:41,190 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T03:17:41,190 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f5c6727-65f7-4659-afe8-8e9a55664c78' 2024-11-10T03:17:41,190 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T03:17:41,190 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f5c6727-65f7-4659-afe8-8e9a55664c78" 2024-11-10T03:17:41,191 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6472108f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:17:41,191 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [394d2222f724,37299,-1] 2024-11-10T03:17:41,191 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T03:17:41,191 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:17:41,192 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40392, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T03:17:41,192 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2773ba76, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:17:41,193 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7999da1b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:17:41,193 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 394d2222f724,37299,-1 for getting cluster id 2024-11-10T03:17:41,193 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T03:17:41,193 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f5c6727-65f7-4659-afe8-8e9a55664c78' 2024-11-10T03:17:41,194 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T03:17:41,194 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f5c6727-65f7-4659-afe8-8e9a55664c78" 2024-11-10T03:17:41,194 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38dfa187, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:17:41,194 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [394d2222f724,37299,-1] 2024-11-10T03:17:41,194 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T03:17:41,194 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:17:41,195 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40402, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T03:17:41,195 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@418f56, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:17:41,196 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@781a5df7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:17:41,196 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 394d2222f724,37299,-1 for getting cluster id 2024-11-10T03:17:41,196 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T03:17:41,197 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f5c6727-65f7-4659-afe8-8e9a55664c78' 2024-11-10T03:17:41,197 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T03:17:41,197 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f5c6727-65f7-4659-afe8-8e9a55664c78" 2024-11-10T03:17:41,197 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44770d35, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:17:41,197 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [394d2222f724,37299,-1] 2024-11-10T03:17:41,198 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T03:17:41,198 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:17:41,199 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40424, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T03:17:41,199 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7db9bed1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:17:41,200 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27da8a0e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:17:41,200 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 394d2222f724,37299,-1 for getting cluster id 2024-11-10T03:17:41,200 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T03:17:41,201 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f5c6727-65f7-4659-afe8-8e9a55664c78' 2024-11-10T03:17:41,201 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T03:17:41,201 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f5c6727-65f7-4659-afe8-8e9a55664c78" 2024-11-10T03:17:41,201 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41504423, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:17:41,201 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [394d2222f724,37299,-1] 2024-11-10T03:17:41,201 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T03:17:41,201 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:17:41,202 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40444, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T03:17:41,202 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67cb67fd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:17:41,203 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@230cc9b5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:17:41,203 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 394d2222f724,37299,-1 for getting cluster id 2024-11-10T03:17:41,203 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T03:17:41,203 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f5c6727-65f7-4659-afe8-8e9a55664c78' 2024-11-10T03:17:41,204 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T03:17:41,204 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f5c6727-65f7-4659-afe8-8e9a55664c78" 2024-11-10T03:17:41,204 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@633f911b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:17:41,204 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [394d2222f724,37299,-1] 2024-11-10T03:17:41,204 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T03:17:41,204 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:17:41,205 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40462, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T03:17:41,205 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39e03627, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:17:41,205 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@122111e0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:17:41,206 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 394d2222f724,37299,-1 for getting cluster id 2024-11-10T03:17:41,206 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T03:17:41,206 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f5c6727-65f7-4659-afe8-8e9a55664c78' 2024-11-10T03:17:41,206 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T03:17:41,206 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f5c6727-65f7-4659-afe8-8e9a55664c78" 2024-11-10T03:17:41,206 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@221c3726, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:17:41,206 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [394d2222f724,37299,-1] 2024-11-10T03:17:41,207 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T03:17:41,207 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:17:41,207 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40474, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T03:17:41,208 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1283bb7e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:17:41,208 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25dafdb8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:17:41,208 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 394d2222f724,37299,-1 for getting cluster id 2024-11-10T03:17:41,209 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T03:17:41,209 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f5c6727-65f7-4659-afe8-8e9a55664c78' 2024-11-10T03:17:41,209 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T03:17:41,209 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f5c6727-65f7-4659-afe8-8e9a55664c78" 2024-11-10T03:17:41,210 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7cfa5c2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:17:41,210 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [394d2222f724,37299,-1] 2024-11-10T03:17:41,210 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T03:17:41,210 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:17:41,210 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40490, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T03:17:41,211 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b0180d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:17:41,211 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@362b1219, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:17:41,211 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 394d2222f724,37299,-1 for getting cluster id 2024-11-10T03:17:41,211 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T03:17:41,212 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f5c6727-65f7-4659-afe8-8e9a55664c78' 2024-11-10T03:17:41,212 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T03:17:41,212 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f5c6727-65f7-4659-afe8-8e9a55664c78" 2024-11-10T03:17:41,212 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e5bf869, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:17:41,212 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [394d2222f724,37299,-1] 2024-11-10T03:17:41,212 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T03:17:41,213 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:17:41,213 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40508, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T03:17:41,214 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@713e4b87, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:17:41,214 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1eef0c91, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:17:41,214 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 394d2222f724,37299,-1 for getting cluster id 2024-11-10T03:17:41,214 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T03:17:41,215 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f5c6727-65f7-4659-afe8-8e9a55664c78' 2024-11-10T03:17:41,215 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T03:17:41,215 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f5c6727-65f7-4659-afe8-8e9a55664c78" 2024-11-10T03:17:41,215 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b3130f1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:17:41,215 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [394d2222f724,37299,-1] 2024-11-10T03:17:41,216 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T03:17:41,216 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:17:41,216 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40530, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T03:17:41,217 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1308c4b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:17:41,217 DEBUG [Thread-1568 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T03:17:41,217 DEBUG [Thread-1557 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T03:17:41,218 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=394d2222f724,38139,1731208572054, seqNum=-1] 2024-11-10T03:17:41,218 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=394d2222f724,38139,1731208572054, seqNum=-1] 2024-11-10T03:17:41,218 DEBUG [Thread-1572 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T03:17:41,218 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T03:17:41,218 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T03:17:41,218 DEBUG [Thread-1564 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T03:17:41,218 DEBUG [Thread-1555 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T03:17:41,218 DEBUG [Thread-1559 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T03:17:41,219 DEBUG [Thread-1566 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T03:17:41,219 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=394d2222f724,38139,1731208572054, seqNum=-1] 2024-11-10T03:17:41,219 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=394d2222f724,38139,1731208572054, seqNum=-1] 2024-11-10T03:17:41,219 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T03:17:41,219 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T03:17:41,220 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=394d2222f724,38139,1731208572054, seqNum=-1] 2024-11-10T03:17:41,220 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T03:17:41,220 DEBUG [Thread-1570 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T03:17:41,220 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53942, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T03:17:41,220 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53926, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T03:17:41,220 DEBUG [Thread-1553 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T03:17:41,220 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53924, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T03:17:41,221 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=394d2222f724,38139,1731208572054, seqNum=-1] 2024-11-10T03:17:41,221 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53944, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T03:17:41,221 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53958, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T03:17:41,221 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=394d2222f724,38139,1731208572054, seqNum=-1] 2024-11-10T03:17:41,221 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T03:17:41,221 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T03:17:41,221 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=394d2222f724,38139,1731208572054, seqNum=-1] 2024-11-10T03:17:41,221 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53968, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T03:17:41,221 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53984, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T03:17:41,222 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T03:17:41,222 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=394d2222f724,38139,1731208572054, seqNum=-1] 2024-11-10T03:17:41,222 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5] 2024-11-10T03:17:41,222 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T03:17:41,222 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53986, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T03:17:41,222 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5] 2024-11-10T03:17:41,223 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53998, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T03:17:41,223 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5] 2024-11-10T03:17:41,224 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5] 2024-11-10T03:17:41,224 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5] 2024-11-10T03:17:41,225 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5] 2024-11-10T03:17:41,225 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5] 2024-11-10T03:17:41,226 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5] 2024-11-10T03:17:41,226 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5] 2024-11-10T03:17:41,228 DEBUG [Thread-1561 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T03:17:41,229 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T03:17:41,229 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=394d2222f724,38139,1731208572054, seqNum=-1] 2024-11-10T03:17:41,229 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T03:17:41,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-11-10T03:17:41,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-10T03:17:41,230 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54002, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T03:17:41,231 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T03:17:41,231 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T03:17:41,231 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T03:17:41,232 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5] 2024-11-10T03:17:41,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on c6879c5307a3686e6731f46d96d3ca08 2024-11-10T03:17:41,297 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c6879c5307a3686e6731f46d96d3ca08 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-10T03:17:41,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6879c5307a3686e6731f46d96d3ca08, store=A 2024-11-10T03:17:41,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:17:41,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6879c5307a3686e6731f46d96d3ca08, store=B 2024-11-10T03:17:41,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:17:41,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6879c5307a3686e6731f46d96d3ca08, store=C 2024-11-10T03:17:41,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:17:41,325 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110ded3a2f9b34640dfac5978f316197b72_c6879c5307a3686e6731f46d96d3ca08 is 50, key is test_row_0/A:col10/1731208661277/Put/seqid=0 2024-11-10T03:17:41,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742167_1343 (size=12154) 2024-11-10T03:17:41,334 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:41,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-10T03:17:41,337 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110ded3a2f9b34640dfac5978f316197b72_c6879c5307a3686e6731f46d96d3ca08 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110ded3a2f9b34640dfac5978f316197b72_c6879c5307a3686e6731f46d96d3ca08 2024-11-10T03:17:41,338 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/A/7fac2dd553654faa9bd537ed3e8b83cc, store: [table=TestAcidGuarantees family=A region=c6879c5307a3686e6731f46d96d3ca08] 2024-11-10T03:17:41,338 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/A/7fac2dd553654faa9bd537ed3e8b83cc is 175, key is test_row_0/A:col10/1731208661277/Put/seqid=0 2024-11-10T03:17:41,340 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:41,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53998 deadline: 1731208721327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:41,340 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:41,340 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:41,340 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:41,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53984 deadline: 1731208721333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:41,341 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:41,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:41,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53958 deadline: 1731208721334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:41,341 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:41,341 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:41,341 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:41,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:41,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:41,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:41,343 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:41,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53924 deadline: 1731208721339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:41,344 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:41,344 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:41,344 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:41,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742168_1344 (size=30955) 2024-11-10T03:17:41,351 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:41,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54002 deadline: 1731208721341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:41,351 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:41,351 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:41,351 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:41,383 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-10T03:17:41,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:41,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. as already flushing 2024-11-10T03:17:41,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:41,384 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:41,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:41,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:41,448 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:41,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53998 deadline: 1731208721443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:41,448 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:41,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53984 deadline: 1731208721443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:41,448 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:41,448 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:41,448 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:41,448 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:41,448 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:41,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53958 deadline: 1731208721443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:41,448 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:41,448 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:41,449 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:41,449 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:41,449 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:41,458 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:41,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53924 deadline: 1731208721452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:41,458 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:41,458 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:41,458 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:41,471 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:41,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54002 deadline: 1731208721463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:41,471 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:41,471 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:41,471 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:41,540 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-10T03:17:41,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:41,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. as already flushing 2024-11-10T03:17:41,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:41,541 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:41,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:41,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:41,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-10T03:17:41,662 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:41,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53984 deadline: 1731208721653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:41,663 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:41,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53958 deadline: 1731208721653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:41,663 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:41,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53998 deadline: 1731208721653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:41,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:41,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:41,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:41,664 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:41,664 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:41,664 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:41,665 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:41,665 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:41,665 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:41,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:41,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53924 deadline: 1731208721665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:41,675 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:41,675 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:41,675 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:41,686 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:41,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54002 deadline: 1731208721683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:41,686 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:41,687 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:41,687 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:41,696 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-10T03:17:41,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:41,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. as already flushing 2024-11-10T03:17:41,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:41,696 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:41,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:41,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:41,749 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/A/7fac2dd553654faa9bd537ed3e8b83cc 2024-11-10T03:17:41,794 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/B/df5dec3c8cf34bc3a636f7880cf6e713 is 50, key is test_row_0/B:col10/1731208661277/Put/seqid=0 2024-11-10T03:17:41,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742169_1345 (size=12001) 2024-11-10T03:17:41,835 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/B/df5dec3c8cf34bc3a636f7880cf6e713 2024-11-10T03:17:41,853 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-10T03:17:41,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-10T03:17:41,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:41,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. as already flushing 2024-11-10T03:17:41,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:41,861 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:41,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:41,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:41,876 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/C/bbb0253fa2a949e7b4b6af00dab1930b is 50, key is test_row_0/C:col10/1731208661277/Put/seqid=0 2024-11-10T03:17:41,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742170_1346 (size=12001) 2024-11-10T03:17:41,911 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/C/bbb0253fa2a949e7b4b6af00dab1930b 2024-11-10T03:17:41,931 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/A/7fac2dd553654faa9bd537ed3e8b83cc as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/7fac2dd553654faa9bd537ed3e8b83cc 2024-11-10T03:17:41,946 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/7fac2dd553654faa9bd537ed3e8b83cc, entries=150, sequenceid=16, filesize=30.2 K 2024-11-10T03:17:41,947 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/B/df5dec3c8cf34bc3a636f7880cf6e713 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/df5dec3c8cf34bc3a636f7880cf6e713 2024-11-10T03:17:41,956 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/df5dec3c8cf34bc3a636f7880cf6e713, entries=150, sequenceid=16, filesize=11.7 K 2024-11-10T03:17:41,957 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/C/bbb0253fa2a949e7b4b6af00dab1930b as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/bbb0253fa2a949e7b4b6af00dab1930b 2024-11-10T03:17:41,967 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/bbb0253fa2a949e7b4b6af00dab1930b, entries=150, sequenceid=16, filesize=11.7 K 2024-11-10T03:17:41,968 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for c6879c5307a3686e6731f46d96d3ca08 in 671ms, sequenceid=16, compaction requested=false 2024-11-10T03:17:41,969 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c6879c5307a3686e6731f46d96d3ca08: 2024-11-10T03:17:41,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on c6879c5307a3686e6731f46d96d3ca08 2024-11-10T03:17:41,982 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c6879c5307a3686e6731f46d96d3ca08 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-10T03:17:41,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6879c5307a3686e6731f46d96d3ca08, store=A 2024-11-10T03:17:41,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T03:17:41,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6879c5307a3686e6731f46d96d3ca08, store=B 2024-11-10T03:17:41,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T03:17:41,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6879c5307a3686e6731f46d96d3ca08, store=C 2024-11-10T03:17:41,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T03:17:42,026 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-10T03:17:42,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:42,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. as already flushing 2024-11-10T03:17:42,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:42,027 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:42,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:42,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:42,056 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111058e38dc638c04304bf60696506ddc0c9_c6879c5307a3686e6731f46d96d3ca08 is 50, key is test_row_0/A:col10/1731208661337/Put/seqid=0 2024-11-10T03:17:42,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:42,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54002 deadline: 1731208722052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:42,067 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:42,067 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:42,067 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:42,067 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:42,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53958 deadline: 1731208722052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:42,067 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:42,067 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:42,067 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:42,077 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:42,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53984 deadline: 1731208722064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:42,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:42,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:42,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:42,082 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:42,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53924 deadline: 1731208722066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:42,085 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:42,085 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:42,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53998 deadline: 1731208722067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:42,085 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:42,085 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:42,085 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:42,086 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:42,086 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:42,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742171_1347 (size=12154) 2024-11-10T03:17:42,089 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:42,112 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111058e38dc638c04304bf60696506ddc0c9_c6879c5307a3686e6731f46d96d3ca08 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111058e38dc638c04304bf60696506ddc0c9_c6879c5307a3686e6731f46d96d3ca08 2024-11-10T03:17:42,115 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/A/842e6f66ebce4bbdab1d5854de7cc815, store: [table=TestAcidGuarantees family=A region=c6879c5307a3686e6731f46d96d3ca08] 2024-11-10T03:17:42,116 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/A/842e6f66ebce4bbdab1d5854de7cc815 is 175, key is test_row_0/A:col10/1731208661337/Put/seqid=0 2024-11-10T03:17:42,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742172_1348 (size=30955) 2024-11-10T03:17:42,143 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=37, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/A/842e6f66ebce4bbdab1d5854de7cc815 2024-11-10T03:17:42,176 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/B/dec5abd1b1174959a025f5d883df45f9 is 50, key is test_row_0/B:col10/1731208661337/Put/seqid=0 2024-11-10T03:17:42,184 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:42,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54002 deadline: 1731208722173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:42,185 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-10T03:17:42,185 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:42,185 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:42,185 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:42,185 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:42,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:42,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53958 deadline: 1731208722173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:42,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. as already flushing 2024-11-10T03:17:42,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:42,185 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:42,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:42,186 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:42,186 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:42,186 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:42,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:42,195 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:42,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53984 deadline: 1731208722184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:42,196 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:42,196 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:42,196 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:42,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742173_1349 (size=12001) 2024-11-10T03:17:42,207 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:42,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53924 deadline: 1731208722193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:42,208 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:42,207 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:42,208 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:42,208 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:42,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53998 deadline: 1731208722193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:42,208 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:42,208 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:42,208 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:42,212 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/B/dec5abd1b1174959a025f5d883df45f9 2024-11-10T03:17:42,275 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/C/6ca0ea000c5049138f13440cbf77ea5b is 50, key is test_row_0/C:col10/1731208661337/Put/seqid=0 2024-11-10T03:17:42,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742174_1350 (size=12001) 2024-11-10T03:17:42,339 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-10T03:17:42,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:42,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. as already flushing 2024-11-10T03:17:42,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:42,340 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:42,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:42,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:42,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-10T03:17:42,403 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:42,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54002 deadline: 1731208722393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:42,403 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:42,403 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:42,403 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:42,404 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:42,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53958 deadline: 1731208722393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:42,405 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:42,405 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:42,405 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:42,409 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:42,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53984 deadline: 1731208722403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:42,410 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:42,410 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:42,410 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:42,427 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:42,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53998 deadline: 1731208722413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:42,428 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:42,428 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:42,428 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:42,428 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:42,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53924 deadline: 1731208722416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:42,429 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:42,429 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:42,429 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:42,493 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-10T03:17:42,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:42,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. as already flushing 2024-11-10T03:17:42,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:42,494 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:42,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:42,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:42,656 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-10T03:17:42,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:42,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. as already flushing 2024-11-10T03:17:42,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:42,657 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:42,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:42,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:42,720 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/C/6ca0ea000c5049138f13440cbf77ea5b 2024-11-10T03:17:42,726 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:42,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54002 deadline: 1731208722716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:42,726 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:42,726 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:42,726 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:42,727 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:42,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53958 deadline: 1731208722716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:42,727 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:42,727 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:42,727 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:42,736 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:42,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53984 deadline: 1731208722723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:42,737 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:42,737 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:42,737 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:42,744 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:42,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53924 deadline: 1731208722733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:42,744 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:42,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53998 deadline: 1731208722735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:42,745 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:42,745 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:42,745 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:42,747 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:42,747 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:42,747 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:42,760 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/A/842e6f66ebce4bbdab1d5854de7cc815 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/842e6f66ebce4bbdab1d5854de7cc815 2024-11-10T03:17:42,801 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/842e6f66ebce4bbdab1d5854de7cc815, entries=150, sequenceid=37, filesize=30.2 K 2024-11-10T03:17:42,807 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/B/dec5abd1b1174959a025f5d883df45f9 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/dec5abd1b1174959a025f5d883df45f9 2024-11-10T03:17:42,812 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-10T03:17:42,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:42,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. as already flushing 2024-11-10T03:17:42,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:42,813 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:42,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:42,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:42,818 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/dec5abd1b1174959a025f5d883df45f9, entries=150, sequenceid=37, filesize=11.7 K 2024-11-10T03:17:42,824 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/C/6ca0ea000c5049138f13440cbf77ea5b as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/6ca0ea000c5049138f13440cbf77ea5b 2024-11-10T03:17:42,830 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/6ca0ea000c5049138f13440cbf77ea5b, entries=150, sequenceid=37, filesize=11.7 K 2024-11-10T03:17:42,831 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for c6879c5307a3686e6731f46d96d3ca08 in 849ms, sequenceid=37, compaction requested=false 2024-11-10T03:17:42,831 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c6879c5307a3686e6731f46d96d3ca08: 2024-11-10T03:17:42,969 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-10T03:17:42,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:42,972 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing c6879c5307a3686e6731f46d96d3ca08 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-10T03:17:42,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6879c5307a3686e6731f46d96d3ca08, store=A 2024-11-10T03:17:42,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:17:42,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6879c5307a3686e6731f46d96d3ca08, store=B 2024-11-10T03:17:42,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:17:42,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6879c5307a3686e6731f46d96d3ca08, store=C 2024-11-10T03:17:42,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:17:42,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411104a30ffbbeb854a859e7328b4cfdaf78c_c6879c5307a3686e6731f46d96d3ca08 is 50, key is test_row_0/A:col10/1731208662035/Put/seqid=0 2024-11-10T03:17:43,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742175_1351 (size=12154) 2024-11-10T03:17:43,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on c6879c5307a3686e6731f46d96d3ca08 2024-11-10T03:17:43,244 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. as already flushing 2024-11-10T03:17:43,299 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:43,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54002 deadline: 1731208723289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:43,300 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:43,300 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:43,300 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:43,301 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731208663264, max timestamp=1731208663298Num uniques 859; 2024-11-10T03:17:43,301 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-10T03:17:43,360 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:43,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53998 deadline: 1731208723317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:43,361 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:43,361 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:43,361 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:43,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:43,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53984 deadline: 1731208723349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:43,367 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:43,367 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:43,368 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:43,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:43,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53958 deadline: 1731208723349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:43,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:43,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:43,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:43,368 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:43,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53924 deadline: 1731208723349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:43,368 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:43,368 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:43,368 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:43,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-10T03:17:43,416 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:43,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54002 deadline: 1731208723403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:43,416 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:43,416 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:43,416 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:43,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:43,426 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411104a30ffbbeb854a859e7328b4cfdaf78c_c6879c5307a3686e6731f46d96d3ca08 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411104a30ffbbeb854a859e7328b4cfdaf78c_c6879c5307a3686e6731f46d96d3ca08 2024-11-10T03:17:43,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/A/06b454b931ba487ebac5af1c649e1a8e, store: [table=TestAcidGuarantees family=A region=c6879c5307a3686e6731f46d96d3ca08] 2024-11-10T03:17:43,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/A/06b454b931ba487ebac5af1c649e1a8e is 175, key is test_row_0/A:col10/1731208662035/Put/seqid=0 2024-11-10T03:17:43,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742176_1352 (size=30955) 2024-11-10T03:17:43,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:43,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53998 deadline: 1731208723462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:43,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:43,475 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:43,475 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:43,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53958 deadline: 1731208723473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:43,475 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:43,476 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:43,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:43,476 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:43,476 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:43,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53984 deadline: 1731208723474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:43,476 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:43,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:43,476 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:43,476 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:43,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53924 deadline: 1731208723474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:43,476 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:43,476 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:43,476 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:43,639 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:43,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54002 deadline: 1731208723624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:43,640 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:43,640 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:43,640 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:43,694 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:43,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53958 deadline: 1731208723683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:43,695 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:43,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53998 deadline: 1731208723683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:43,695 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:43,695 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:43,695 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:43,695 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:43,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53984 deadline: 1731208723683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:43,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:43,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:43,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:43,695 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:43,696 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:43,696 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:43,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53924 deadline: 1731208723683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:43,696 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:43,696 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:43,696 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:43,696 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:43,807 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-10T03:17:43,856 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=48, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/A/06b454b931ba487ebac5af1c649e1a8e 2024-11-10T03:17:43,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/B/92cb5392a2eb4b88bacf4d0f8a14b538 is 50, key is test_row_0/B:col10/1731208662035/Put/seqid=0 2024-11-10T03:17:43,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:43,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54002 deadline: 1731208723958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:43,969 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:43,969 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:43,969 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:43,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742177_1353 (size=12001) 2024-11-10T03:17:44,012 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:44,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53998 deadline: 1731208724003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:44,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:44,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53958 deadline: 1731208724003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:44,013 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:44,013 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:44,013 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:44,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:44,013 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:44,013 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:44,013 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:44,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53984 deadline: 1731208724003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:44,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:44,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:44,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:44,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:44,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53924 deadline: 1731208724004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:44,014 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:44,014 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:44,014 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:44,376 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/B/92cb5392a2eb4b88bacf4d0f8a14b538 2024-11-10T03:17:44,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/C/61048f5ffb7e4985a77c30b50d2a0cb6 is 50, key is test_row_0/C:col10/1731208662035/Put/seqid=0 2024-11-10T03:17:44,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742178_1354 (size=12001) 2024-11-10T03:17:44,454 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/C/61048f5ffb7e4985a77c30b50d2a0cb6 2024-11-10T03:17:44,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/A/06b454b931ba487ebac5af1c649e1a8e as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/06b454b931ba487ebac5af1c649e1a8e 2024-11-10T03:17:44,490 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/06b454b931ba487ebac5af1c649e1a8e, entries=150, sequenceid=48, filesize=30.2 K 2024-11-10T03:17:44,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/B/92cb5392a2eb4b88bacf4d0f8a14b538 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/92cb5392a2eb4b88bacf4d0f8a14b538 2024-11-10T03:17:44,513 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/92cb5392a2eb4b88bacf4d0f8a14b538, entries=150, sequenceid=48, filesize=11.7 K 2024-11-10T03:17:44,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/C/61048f5ffb7e4985a77c30b50d2a0cb6 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/61048f5ffb7e4985a77c30b50d2a0cb6 2024-11-10T03:17:44,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:44,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54002 deadline: 1731208724484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:44,515 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:44,515 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:44,515 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:44,526 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/61048f5ffb7e4985a77c30b50d2a0cb6, entries=150, sequenceid=48, filesize=11.7 K 2024-11-10T03:17:44,532 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for c6879c5307a3686e6731f46d96d3ca08 in 1560ms, sequenceid=48, compaction requested=true 2024-11-10T03:17:44,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for c6879c5307a3686e6731f46d96d3ca08: 2024-11-10T03:17:44,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:44,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-11-10T03:17:44,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-11-10T03:17:44,543 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c6879c5307a3686e6731f46d96d3ca08 3/3 column families, dataSize=134.18 KB heapSize=391.80 KB 2024-11-10T03:17:44,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6879c5307a3686e6731f46d96d3ca08, store=A 2024-11-10T03:17:44,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T03:17:44,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6879c5307a3686e6731f46d96d3ca08, store=B 2024-11-10T03:17:44,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T03:17:44,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6879c5307a3686e6731f46d96d3ca08, store=C 2024-11-10T03:17:44,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T03:17:44,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on c6879c5307a3686e6731f46d96d3ca08 2024-11-10T03:17:44,548 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-10T03:17:44,549 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.3060 sec 2024-11-10T03:17:44,553 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 3.3210 sec 2024-11-10T03:17:44,556 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411102f7b65edc96343de8af7009a16bae449_c6879c5307a3686e6731f46d96d3ca08 is 50, key is test_row_0/A:col10/1731208664541/Put/seqid=0 2024-11-10T03:17:44,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742179_1355 (size=14594) 2024-11-10T03:17:44,591 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:44,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53998 deadline: 1731208724558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:44,592 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:44,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53958 deadline: 1731208724560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:44,592 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:44,593 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:44,593 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:44,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:44,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53984 deadline: 1731208724574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:44,595 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:44,595 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:44,595 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:44,595 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:44,595 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:44,595 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:44,616 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:44,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53924 deadline: 1731208724591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:44,616 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:44,617 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:44,617 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:44,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:44,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53958 deadline: 1731208724703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:44,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:44,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53998 deadline: 1731208724703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:44,719 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:44,719 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:44,719 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:44,719 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:44,719 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:44,719 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:44,719 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:44,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53984 deadline: 1731208724704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:44,724 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:44,725 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:44,725 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:44,740 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:44,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53924 deadline: 1731208724724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:44,744 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:44,744 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:44,744 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:44,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:44,946 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:44,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53998 deadline: 1731208724928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:44,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53958 deadline: 1731208724925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:44,947 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:44,947 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:44,947 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:44,947 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:44,947 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:44,947 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:44,947 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:44,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53984 deadline: 1731208724933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:44,948 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:44,948 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:44,948 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:44,976 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:44,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53924 deadline: 1731208724954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:44,977 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:44,978 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:44,978 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:44,982 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:45,009 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411102f7b65edc96343de8af7009a16bae449_c6879c5307a3686e6731f46d96d3ca08 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411102f7b65edc96343de8af7009a16bae449_c6879c5307a3686e6731f46d96d3ca08 2024-11-10T03:17:45,021 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/A/b432b2aa53a44f9bb188b32cb5d53dae, store: [table=TestAcidGuarantees family=A region=c6879c5307a3686e6731f46d96d3ca08] 2024-11-10T03:17:45,021 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/A/b432b2aa53a44f9bb188b32cb5d53dae is 175, key is test_row_0/A:col10/1731208664541/Put/seqid=0 2024-11-10T03:17:45,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742180_1356 (size=39549) 2024-11-10T03:17:45,261 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:45,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53998 deadline: 1731208725253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:45,262 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:45,262 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:45,262 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:45,262 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:45,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53958 deadline: 1731208725254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:45,263 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:45,263 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:45,263 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:45,263 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:45,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53984 deadline: 1731208725256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:45,264 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:45,264 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:45,264 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:45,296 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:45,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53924 deadline: 1731208725283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:45,297 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:45,297 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:45,297 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:45,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-10T03:17:45,408 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T03:17:45,412 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T03:17:45,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-11-10T03:17:45,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-10T03:17:45,420 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T03:17:45,423 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T03:17:45,423 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T03:17:45,469 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=75, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/A/b432b2aa53a44f9bb188b32cb5d53dae 2024-11-10T03:17:45,482 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/B/fdebc8214ddb447493f904b4773c1d0c is 50, key is test_row_0/B:col10/1731208664541/Put/seqid=0 2024-11-10T03:17:45,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742181_1357 (size=12001) 2024-11-10T03:17:45,517 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/B/fdebc8214ddb447493f904b4773c1d0c 2024-11-10T03:17:45,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-10T03:17:45,529 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:45,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54002 deadline: 1731208725523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:45,531 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 394d2222f724,38139,1731208572054 for 'test_row_0' in c6879c5307a3686e6731f46d96d3ca08 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2242 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T03:17:45,532 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:45,532 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:45,532 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:45,547 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/C/73161c86224746bba1de7c1fa2a61b8d is 50, key is test_row_0/C:col10/1731208664541/Put/seqid=0 2024-11-10T03:17:45,589 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-10T03:17:45,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:45,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. as already flushing 2024-11-10T03:17:45,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:45,589 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:45,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:45,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:45,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742182_1358 (size=12001) 2024-11-10T03:17:45,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-10T03:17:45,748 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-10T03:17:45,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:45,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. as already flushing 2024-11-10T03:17:45,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:45,749 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:45,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:45,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:45,778 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:45,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53958 deadline: 1731208725773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:45,778 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:45,778 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:45,778 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:45,778 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:45,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53998 deadline: 1731208725773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:45,779 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:45,779 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:45,779 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:45,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:45,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53984 deadline: 1731208725774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:45,779 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:45,779 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:45,779 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:45,832 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:45,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53924 deadline: 1731208725813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:45,832 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:45,832 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:45,832 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:45,902 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-10T03:17:45,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:45,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. as already flushing 2024-11-10T03:17:45,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:45,902 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:45,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:45,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:46,018 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/C/73161c86224746bba1de7c1fa2a61b8d 2024-11-10T03:17:46,024 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/A/b432b2aa53a44f9bb188b32cb5d53dae as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/b432b2aa53a44f9bb188b32cb5d53dae 2024-11-10T03:17:46,033 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/b432b2aa53a44f9bb188b32cb5d53dae, entries=200, sequenceid=75, filesize=38.6 K 2024-11-10T03:17:46,034 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/B/fdebc8214ddb447493f904b4773c1d0c as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/fdebc8214ddb447493f904b4773c1d0c 2024-11-10T03:17:46,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,038 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/fdebc8214ddb447493f904b4773c1d0c, entries=150, sequenceid=75, filesize=11.7 K 2024-11-10T03:17:46,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,038 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/C/73161c86224746bba1de7c1fa2a61b8d as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/73161c86224746bba1de7c1fa2a61b8d 2024-11-10T03:17:46,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,043 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/73161c86224746bba1de7c1fa2a61b8d, entries=150, sequenceid=75, filesize=11.7 K 2024-11-10T03:17:46,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-10T03:17:46,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,044 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~391.75 KB/401152, currentSize=40.25 KB/41220 for c6879c5307a3686e6731f46d96d3ca08 in 1501ms, sequenceid=75, compaction requested=true 2024-11-10T03:17:46,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,045 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c6879c5307a3686e6731f46d96d3ca08: 2024-11-10T03:17:46,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,045 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T03:17:46,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c6879c5307a3686e6731f46d96d3ca08:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T03:17:46,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:17:46,046 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T03:17:46,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,046 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 132414 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T03:17:46,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,047 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1541): c6879c5307a3686e6731f46d96d3ca08/A is initiating minor compaction (all files) 2024-11-10T03:17:46,047 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c6879c5307a3686e6731f46d96d3ca08/A in TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:46,047 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/7fac2dd553654faa9bd537ed3e8b83cc, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/842e6f66ebce4bbdab1d5854de7cc815, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/06b454b931ba487ebac5af1c649e1a8e, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/b432b2aa53a44f9bb188b32cb5d53dae] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp, totalSize=129.3 K 2024-11-10T03:17:46,047 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:46,047 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. files: [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/7fac2dd553654faa9bd537ed3e8b83cc, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/842e6f66ebce4bbdab1d5854de7cc815, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/06b454b931ba487ebac5af1c649e1a8e, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/b432b2aa53a44f9bb188b32cb5d53dae] 2024-11-10T03:17:46,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,047 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7fac2dd553654faa9bd537ed3e8b83cc, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731208661270 2024-11-10T03:17:46,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,047 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c6879c5307a3686e6731f46d96d3ca08:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T03:17:46,047 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T03:17:46,047 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:17:46,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c6879c5307a3686e6731f46d96d3ca08:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T03:17:46,048 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1541): c6879c5307a3686e6731f46d96d3ca08/B is initiating minor compaction (all files) 2024-11-10T03:17:46,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T03:17:46,048 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c6879c5307a3686e6731f46d96d3ca08/B in TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:46,048 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/df5dec3c8cf34bc3a636f7880cf6e713, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/dec5abd1b1174959a025f5d883df45f9, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/92cb5392a2eb4b88bacf4d0f8a14b538, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/fdebc8214ddb447493f904b4773c1d0c] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp, totalSize=46.9 K 2024-11-10T03:17:46,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,048 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 842e6f66ebce4bbdab1d5854de7cc815, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731208661337 2024-11-10T03:17:46,048 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting df5dec3c8cf34bc3a636f7880cf6e713, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731208661270 2024-11-10T03:17:46,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,048 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 06b454b931ba487ebac5af1c649e1a8e, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731208662035 2024-11-10T03:17:46,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,048 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting dec5abd1b1174959a025f5d883df45f9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731208661337 2024-11-10T03:17:46,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,049 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 92cb5392a2eb4b88bacf4d0f8a14b538, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731208662035 2024-11-10T03:17:46,049 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting b432b2aa53a44f9bb188b32cb5d53dae, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1731208663314 2024-11-10T03:17:46,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,049 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting fdebc8214ddb447493f904b4773c1d0c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1731208663314 2024-11-10T03:17:46,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,055 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-10T03:17:46,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:46,055 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing c6879c5307a3686e6731f46d96d3ca08 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-10T03:17:46,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6879c5307a3686e6731f46d96d3ca08, store=A 2024-11-10T03:17:46,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:17:46,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6879c5307a3686e6731f46d96d3ca08, store=B 2024-11-10T03:17:46,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:17:46,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6879c5307a3686e6731f46d96d3ca08, store=C 2024-11-10T03:17:46,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:17:46,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,057 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c6879c5307a3686e6731f46d96d3ca08] 2024-11-10T03:17:46,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,065 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241110387a499763ca4deb9575d7c15df310d0_c6879c5307a3686e6731f46d96d3ca08 store=[table=TestAcidGuarantees family=A region=c6879c5307a3686e6731f46d96d3ca08] 2024-11-10T03:17:46,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,066 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241110387a499763ca4deb9575d7c15df310d0_c6879c5307a3686e6731f46d96d3ca08, store=[table=TestAcidGuarantees family=A region=c6879c5307a3686e6731f46d96d3ca08] 2024-11-10T03:17:46,066 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110387a499763ca4deb9575d7c15df310d0_c6879c5307a3686e6731f46d96d3ca08 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c6879c5307a3686e6731f46d96d3ca08] 2024-11-10T03:17:46,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,070 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c6879c5307a3686e6731f46d96d3ca08#B#compaction#304 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:17:46,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411106676d18bb2644359a4522ac9f9923ca9_c6879c5307a3686e6731f46d96d3ca08 is 50, key is test_row_0/A:col10/1731208664564/Put/seqid=0 2024-11-10T03:17:46,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,071 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/B/4ee332dae6854afe97843e4d8b72c2ff is 50, key is test_row_0/B:col10/1731208664541/Put/seqid=0 2024-11-10T03:17:46,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742184_1360 (size=9714) 2024-11-10T03:17:46,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742185_1361 (size=12139) 2024-11-10T03:17:46,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742183_1359 (size=4469) 2024-11-10T03:17:46,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,499 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/B/4ee332dae6854afe97843e4d8b72c2ff as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/4ee332dae6854afe97843e4d8b72c2ff 2024-11-10T03:17:46,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,500 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411106676d18bb2644359a4522ac9f9923ca9_c6879c5307a3686e6731f46d96d3ca08 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411106676d18bb2644359a4522ac9f9923ca9_c6879c5307a3686e6731f46d96d3ca08 2024-11-10T03:17:46,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/A/6a0fdfac12404b6f9913d0e9a3878427, store: [table=TestAcidGuarantees family=A region=c6879c5307a3686e6731f46d96d3ca08] 2024-11-10T03:17:46,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/A/6a0fdfac12404b6f9913d0e9a3878427 is 175, key is test_row_0/A:col10/1731208664564/Put/seqid=0 2024-11-10T03:17:46,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,512 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c6879c5307a3686e6731f46d96d3ca08/B of c6879c5307a3686e6731f46d96d3ca08 into 4ee332dae6854afe97843e4d8b72c2ff(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:17:46,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,512 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c6879c5307a3686e6731f46d96d3ca08: 2024-11-10T03:17:46,512 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., storeName=c6879c5307a3686e6731f46d96d3ca08/B, priority=12, startTime=1731208666046; duration=0sec 2024-11-10T03:17:46,512 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T03:17:46,512 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c6879c5307a3686e6731f46d96d3ca08:B 2024-11-10T03:17:46,512 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T03:17:46,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,513 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c6879c5307a3686e6731f46d96d3ca08#A#compaction#303 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:17:46,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,514 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/A/f04ea5eb8d324cc9a854b752d1a0fcdd is 175, key is test_row_0/A:col10/1731208664541/Put/seqid=0 2024-11-10T03:17:46,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,520 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T03:17:46,520 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1541): c6879c5307a3686e6731f46d96d3ca08/C is initiating minor compaction (all files) 2024-11-10T03:17:46,520 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c6879c5307a3686e6731f46d96d3ca08/C in TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:46,520 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/bbb0253fa2a949e7b4b6af00dab1930b, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/6ca0ea000c5049138f13440cbf77ea5b, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/61048f5ffb7e4985a77c30b50d2a0cb6, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/73161c86224746bba1de7c1fa2a61b8d] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp, totalSize=46.9 K 2024-11-10T03:17:46,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,522 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting bbb0253fa2a949e7b4b6af00dab1930b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731208661270 2024-11-10T03:17:46,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,523 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ca0ea000c5049138f13440cbf77ea5b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731208661337 2024-11-10T03:17:46,523 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 61048f5ffb7e4985a77c30b50d2a0cb6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731208662035 2024-11-10T03:17:46,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,523 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 73161c86224746bba1de7c1fa2a61b8d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1731208663314 2024-11-10T03:17:46,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742186_1362 (size=22361) 2024-11-10T03:17:46,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742187_1363 (size=31093) 2024-11-10T03:17:46,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-10T03:17:46,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,559 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c6879c5307a3686e6731f46d96d3ca08#C#compaction#306 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:17:46,559 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/C/5461ea9806454b5e9d1893bb55467690 is 50, key is test_row_0/C:col10/1731208664541/Put/seqid=0 2024-11-10T03:17:46,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742188_1364 (size=12139) 2024-11-10T03:17:46,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,624 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/C/5461ea9806454b5e9d1893bb55467690 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/5461ea9806454b5e9d1893bb55467690 2024-11-10T03:17:46,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,646 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c6879c5307a3686e6731f46d96d3ca08/C of c6879c5307a3686e6731f46d96d3ca08 into 5461ea9806454b5e9d1893bb55467690(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:17:46,646 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c6879c5307a3686e6731f46d96d3ca08: 2024-11-10T03:17:46,646 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., storeName=c6879c5307a3686e6731f46d96d3ca08/C, priority=12, startTime=1731208666048; duration=0sec 2024-11-10T03:17:46,646 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:17:46,646 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c6879c5307a3686e6731f46d96d3ca08:C 2024-11-10T03:17:46,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on c6879c5307a3686e6731f46d96d3ca08 2024-11-10T03:17:46,848 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. as already flushing 2024-11-10T03:17:46,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:46,965 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=84, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/A/6a0fdfac12404b6f9913d0e9a3878427 2024-11-10T03:17:46,979 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/A/f04ea5eb8d324cc9a854b752d1a0fcdd as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/f04ea5eb8d324cc9a854b752d1a0fcdd 2024-11-10T03:17:46,985 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c6879c5307a3686e6731f46d96d3ca08/A of c6879c5307a3686e6731f46d96d3ca08 into f04ea5eb8d324cc9a854b752d1a0fcdd(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:17:46,985 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c6879c5307a3686e6731f46d96d3ca08: 2024-11-10T03:17:46,985 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., storeName=c6879c5307a3686e6731f46d96d3ca08/A, priority=12, startTime=1731208666045; duration=0sec 2024-11-10T03:17:46,985 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:17:46,985 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c6879c5307a3686e6731f46d96d3ca08:A 2024-11-10T03:17:46,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/B/20c205c8ee5d4744b44565541af3cba8 is 50, key is test_row_0/B:col10/1731208664564/Put/seqid=0 2024-11-10T03:17:47,011 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731208666967, max timestamp=1731208667005Num uniques 859; 2024-11-10T03:17:47,011 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T03:17:47,013 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731208666967, max timestamp=1731208667005Num uniques 859; 2024-11-10T03:17:47,013 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T03:17:47,015 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731208666967, max timestamp=1731208667005Num uniques 859; 2024-11-10T03:17:47,015 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T03:17:47,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742189_1365 (size=9657) 2024-11-10T03:17:47,019 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/B/20c205c8ee5d4744b44565541af3cba8 2024-11-10T03:17:47,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/C/d27ca857e4b9450a952d0dd2a096db74 is 50, key is test_row_0/C:col10/1731208664564/Put/seqid=0 2024-11-10T03:17:47,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742190_1366 (size=9657) 2024-11-10T03:17:47,057 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/C/d27ca857e4b9450a952d0dd2a096db74 2024-11-10T03:17:47,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/A/6a0fdfac12404b6f9913d0e9a3878427 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/6a0fdfac12404b6f9913d0e9a3878427 2024-11-10T03:17:47,072 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/6a0fdfac12404b6f9913d0e9a3878427, entries=100, sequenceid=84, filesize=21.8 K 2024-11-10T03:17:47,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/B/20c205c8ee5d4744b44565541af3cba8 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/20c205c8ee5d4744b44565541af3cba8 2024-11-10T03:17:47,078 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/20c205c8ee5d4744b44565541af3cba8, entries=100, sequenceid=84, filesize=9.4 K 2024-11-10T03:17:47,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/C/d27ca857e4b9450a952d0dd2a096db74 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/d27ca857e4b9450a952d0dd2a096db74 2024-11-10T03:17:47,084 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/d27ca857e4b9450a952d0dd2a096db74, entries=100, sequenceid=84, filesize=9.4 K 2024-11-10T03:17:47,085 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=73.80 KB/75570 for c6879c5307a3686e6731f46d96d3ca08 in 1029ms, sequenceid=84, compaction requested=false 2024-11-10T03:17:47,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for c6879c5307a3686e6731f46d96d3ca08: 2024-11-10T03:17:47,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:47,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-11-10T03:17:47,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-11-10T03:17:47,089 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-10T03:17:47,089 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6630 sec 2024-11-10T03:17:47,092 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 1.6780 sec 2024-11-10T03:17:47,093 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c6879c5307a3686e6731f46d96d3ca08 3/3 column families, dataSize=80.51 KB heapSize=220.92 KB 2024-11-10T03:17:47,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6879c5307a3686e6731f46d96d3ca08, store=A 2024-11-10T03:17:47,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T03:17:47,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6879c5307a3686e6731f46d96d3ca08, store=B 2024-11-10T03:17:47,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T03:17:47,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6879c5307a3686e6731f46d96d3ca08, store=C 2024-11-10T03:17:47,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T03:17:47,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on c6879c5307a3686e6731f46d96d3ca08 2024-11-10T03:17:47,130 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110540d8536d8144f8ca4d6bca4cfc0783f_c6879c5307a3686e6731f46d96d3ca08 is 50, key is test_row_0/A:col10/1731208667057/Put/seqid=0 2024-11-10T03:17:47,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742191_1367 (size=14594) 2024-11-10T03:17:47,159 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:47,163 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110540d8536d8144f8ca4d6bca4cfc0783f_c6879c5307a3686e6731f46d96d3ca08 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110540d8536d8144f8ca4d6bca4cfc0783f_c6879c5307a3686e6731f46d96d3ca08 2024-11-10T03:17:47,169 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/A/31582fe8a28a4840aa7bc430a4b44ce8, store: [table=TestAcidGuarantees family=A region=c6879c5307a3686e6731f46d96d3ca08] 2024-11-10T03:17:47,170 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/A/31582fe8a28a4840aa7bc430a4b44ce8 is 175, key is test_row_0/A:col10/1731208667057/Put/seqid=0 2024-11-10T03:17:47,191 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:47,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53958 deadline: 1731208727170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:47,193 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:47,193 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:47,193 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:47,202 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:47,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53998 deadline: 1731208727183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:47,203 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:47,203 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:47,203 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:47,206 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:47,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53984 deadline: 1731208727190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:47,207 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:47,207 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:47,207 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:47,207 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:47,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53924 deadline: 1731208727191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:47,208 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:47,208 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:47,208 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:47,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742192_1368 (size=39549) 2024-11-10T03:17:47,227 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=115, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/A/31582fe8a28a4840aa7bc430a4b44ce8 2024-11-10T03:17:47,254 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/B/11e071d35d634d16892467fd4fdd73d3 is 50, key is test_row_0/B:col10/1731208667057/Put/seqid=0 2024-11-10T03:17:47,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742193_1369 (size=12001) 2024-11-10T03:17:47,315 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:47,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53958 deadline: 1731208727305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:47,316 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:47,316 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:47,316 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:47,316 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:47,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53998 deadline: 1731208727313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:47,317 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:47,317 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:47,317 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:47,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:47,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53984 deadline: 1731208727313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:47,317 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:47,317 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:47,318 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:47,329 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:47,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53924 deadline: 1731208727316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:47,332 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:47,332 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:47,332 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:47,536 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:47,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53958 deadline: 1731208727523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:47,537 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:47,537 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:47,537 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:47,537 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:47,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53998 deadline: 1731208727523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:47,538 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:47,538 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:47,538 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:47,538 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:47,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53984 deadline: 1731208727531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:47,538 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:47,539 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:47,539 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:47,543 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:47,543 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:47,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54002 deadline: 1731208727543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:47,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53924 deadline: 1731208727543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:47,544 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:47,544 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:47,544 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:47,544 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 394d2222f724,38139,1731208572054 for 'test_row_0' in c6879c5307a3686e6731f46d96d3ca08 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4254 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T03:17:47,545 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:47,545 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:47,545 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:47,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-10T03:17:47,564 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T03:17:47,568 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T03:17:47,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-10T03:17:47,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-10T03:17:47,573 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T03:17:47,573 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T03:17:47,574 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T03:17:47,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-10T03:17:47,695 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/B/11e071d35d634d16892467fd4fdd73d3 2024-11-10T03:17:47,718 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/C/dc1026052638488986a9580a91a04fc4 is 50, key is test_row_0/C:col10/1731208667057/Put/seqid=0 2024-11-10T03:17:47,740 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-10T03:17:47,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:47,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. as already flushing 2024-11-10T03:17:47,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:47,741 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:47,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:47,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:47,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742194_1370 (size=12001) 2024-11-10T03:17:47,788 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/C/dc1026052638488986a9580a91a04fc4 2024-11-10T03:17:47,807 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/A/31582fe8a28a4840aa7bc430a4b44ce8 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/31582fe8a28a4840aa7bc430a4b44ce8 2024-11-10T03:17:47,818 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/31582fe8a28a4840aa7bc430a4b44ce8, entries=200, sequenceid=115, filesize=38.6 K 2024-11-10T03:17:47,824 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/B/11e071d35d634d16892467fd4fdd73d3 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/11e071d35d634d16892467fd4fdd73d3 2024-11-10T03:17:47,833 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/11e071d35d634d16892467fd4fdd73d3, entries=150, sequenceid=115, filesize=11.7 K 2024-11-10T03:17:47,834 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/C/dc1026052638488986a9580a91a04fc4 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/dc1026052638488986a9580a91a04fc4 2024-11-10T03:17:47,838 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/dc1026052638488986a9580a91a04fc4, entries=150, sequenceid=115, filesize=11.7 K 2024-11-10T03:17:47,840 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~220.88 KB/226176, currentSize=100.63 KB/103050 for c6879c5307a3686e6731f46d96d3ca08 in 747ms, sequenceid=115, compaction requested=true 2024-11-10T03:17:47,841 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c6879c5307a3686e6731f46d96d3ca08: 2024-11-10T03:17:47,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c6879c5307a3686e6731f46d96d3ca08:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T03:17:47,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:17:47,841 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T03:17:47,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c6879c5307a3686e6731f46d96d3ca08:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T03:17:47,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T03:17:47,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c6879c5307a3686e6731f46d96d3ca08:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T03:17:47,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-10T03:17:47,841 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T03:17:47,844 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T03:17:47,844 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33797 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T03:17:47,844 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1541): c6879c5307a3686e6731f46d96d3ca08/B is initiating minor compaction (all files) 2024-11-10T03:17:47,844 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1541): c6879c5307a3686e6731f46d96d3ca08/A is initiating minor compaction (all files) 2024-11-10T03:17:47,844 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c6879c5307a3686e6731f46d96d3ca08/A in TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:47,844 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c6879c5307a3686e6731f46d96d3ca08/B in TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:47,844 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/f04ea5eb8d324cc9a854b752d1a0fcdd, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/6a0fdfac12404b6f9913d0e9a3878427, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/31582fe8a28a4840aa7bc430a4b44ce8] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp, totalSize=90.8 K 2024-11-10T03:17:47,844 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/4ee332dae6854afe97843e4d8b72c2ff, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/20c205c8ee5d4744b44565541af3cba8, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/11e071d35d634d16892467fd4fdd73d3] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp, totalSize=33.0 K 2024-11-10T03:17:47,844 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:47,844 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. files: [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/f04ea5eb8d324cc9a854b752d1a0fcdd, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/6a0fdfac12404b6f9913d0e9a3878427, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/31582fe8a28a4840aa7bc430a4b44ce8] 2024-11-10T03:17:47,848 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ee332dae6854afe97843e4d8b72c2ff, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1731208663314 2024-11-10T03:17:47,848 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting f04ea5eb8d324cc9a854b752d1a0fcdd, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1731208663314 2024-11-10T03:17:47,848 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 20c205c8ee5d4744b44565541af3cba8, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1731208664564 2024-11-10T03:17:47,849 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a0fdfac12404b6f9913d0e9a3878427, keycount=100, bloomtype=ROW, size=21.8 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1731208664564 2024-11-10T03:17:47,849 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 11e071d35d634d16892467fd4fdd73d3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731208667003 2024-11-10T03:17:47,849 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 31582fe8a28a4840aa7bc430a4b44ce8, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731208667003 2024-11-10T03:17:47,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on c6879c5307a3686e6731f46d96d3ca08 2024-11-10T03:17:47,857 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c6879c5307a3686e6731f46d96d3ca08 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-10T03:17:47,861 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c6879c5307a3686e6731f46d96d3ca08] 2024-11-10T03:17:47,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6879c5307a3686e6731f46d96d3ca08, store=A 2024-11-10T03:17:47,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:17:47,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6879c5307a3686e6731f46d96d3ca08, store=B 2024-11-10T03:17:47,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:17:47,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6879c5307a3686e6731f46d96d3ca08, store=C 2024-11-10T03:17:47,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:17:47,878 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c6879c5307a3686e6731f46d96d3ca08#B#compaction#313 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:17:47,879 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/B/ac5d3b6f752645aaa8e433a0d1b627d0 is 50, key is test_row_0/B:col10/1731208667057/Put/seqid=0 2024-11-10T03:17:47,891 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411100783e90c9cd64aa994dc5ca9fce5cade_c6879c5307a3686e6731f46d96d3ca08 store=[table=TestAcidGuarantees family=A region=c6879c5307a3686e6731f46d96d3ca08] 2024-11-10T03:17:47,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-10T03:17:47,902 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-10T03:17:47,902 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411100783e90c9cd64aa994dc5ca9fce5cade_c6879c5307a3686e6731f46d96d3ca08, store=[table=TestAcidGuarantees family=A region=c6879c5307a3686e6731f46d96d3ca08] 2024-11-10T03:17:47,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:47,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. as already flushing 2024-11-10T03:17:47,903 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411100783e90c9cd64aa994dc5ca9fce5cade_c6879c5307a3686e6731f46d96d3ca08 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c6879c5307a3686e6731f46d96d3ca08] 2024-11-10T03:17:47,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:47,903 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:47,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:47,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:47,912 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111021dd711d3c354386b2c5fe57452a943a_c6879c5307a3686e6731f46d96d3ca08 is 50, key is test_row_0/A:col10/1731208667128/Put/seqid=0 2024-11-10T03:17:47,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742195_1371 (size=12241) 2024-11-10T03:17:47,927 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:47,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53924 deadline: 1731208727897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:47,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:47,928 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:47,928 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:47,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53958 deadline: 1731208727897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:47,928 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:47,929 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:47,929 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:47,929 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:47,933 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:47,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53998 deadline: 1731208727913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:47,933 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:47,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53984 deadline: 1731208727918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:47,934 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:47,934 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:47,934 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:47,936 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:47,936 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:47,936 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:47,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742196_1372 (size=4469) 2024-11-10T03:17:47,942 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c6879c5307a3686e6731f46d96d3ca08#A#compaction#312 average throughput is 0.30 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:17:47,943 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/A/9d701ae590b2482aac0ee6fcdb89bac9 is 175, key is test_row_0/A:col10/1731208667057/Put/seqid=0 2024-11-10T03:17:47,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742197_1373 (size=14744) 2024-11-10T03:17:47,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742198_1374 (size=31195) 2024-11-10T03:17:48,003 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/A/9d701ae590b2482aac0ee6fcdb89bac9 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/9d701ae590b2482aac0ee6fcdb89bac9 2024-11-10T03:17:48,014 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c6879c5307a3686e6731f46d96d3ca08/A of c6879c5307a3686e6731f46d96d3ca08 into 9d701ae590b2482aac0ee6fcdb89bac9(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:17:48,014 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c6879c5307a3686e6731f46d96d3ca08: 2024-11-10T03:17:48,014 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., storeName=c6879c5307a3686e6731f46d96d3ca08/A, priority=13, startTime=1731208667841; duration=0sec 2024-11-10T03:17:48,015 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T03:17:48,015 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c6879c5307a3686e6731f46d96d3ca08:A 2024-11-10T03:17:48,015 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T03:17:48,016 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33797 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T03:17:48,016 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1541): c6879c5307a3686e6731f46d96d3ca08/C is initiating minor compaction (all files) 2024-11-10T03:17:48,016 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c6879c5307a3686e6731f46d96d3ca08/C in TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:48,017 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/5461ea9806454b5e9d1893bb55467690, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/d27ca857e4b9450a952d0dd2a096db74, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/dc1026052638488986a9580a91a04fc4] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp, totalSize=33.0 K 2024-11-10T03:17:48,021 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5461ea9806454b5e9d1893bb55467690, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1731208663314 2024-11-10T03:17:48,022 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting d27ca857e4b9450a952d0dd2a096db74, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1731208664564 2024-11-10T03:17:48,022 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc1026052638488986a9580a91a04fc4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731208667003 2024-11-10T03:17:48,042 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c6879c5307a3686e6731f46d96d3ca08#C#compaction#315 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:17:48,043 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/C/6a2ee4fda36f443ea25a79724d3b17bd is 50, key is test_row_0/C:col10/1731208667057/Put/seqid=0 2024-11-10T03:17:48,049 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:48,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53924 deadline: 1731208728033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:48,049 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:48,049 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:48,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53958 deadline: 1731208728033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:48,049 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:48,049 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:48,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:48,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:48,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:48,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:48,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53984 deadline: 1731208728043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:48,050 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:48,050 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:48,050 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:48,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:48,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53998 deadline: 1731208728043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:48,051 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:48,051 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:48,051 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:48,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742199_1375 (size=12241) 2024-11-10T03:17:48,062 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-10T03:17:48,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:48,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. as already flushing 2024-11-10T03:17:48,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:48,062 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:48,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:48,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:48,074 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/C/6a2ee4fda36f443ea25a79724d3b17bd as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/6a2ee4fda36f443ea25a79724d3b17bd 2024-11-10T03:17:48,082 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c6879c5307a3686e6731f46d96d3ca08/C of c6879c5307a3686e6731f46d96d3ca08 into 6a2ee4fda36f443ea25a79724d3b17bd(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:17:48,082 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c6879c5307a3686e6731f46d96d3ca08: 2024-11-10T03:17:48,082 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., storeName=c6879c5307a3686e6731f46d96d3ca08/C, priority=13, startTime=1731208667841; duration=0sec 2024-11-10T03:17:48,082 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:17:48,082 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c6879c5307a3686e6731f46d96d3ca08:C 2024-11-10T03:17:48,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-10T03:17:48,217 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-10T03:17:48,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:48,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. as already flushing 2024-11-10T03:17:48,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:48,217 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:48,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:48,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:48,264 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:48,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53924 deadline: 1731208728253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:48,264 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:48,264 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:48,264 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:48,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53958 deadline: 1731208728253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:48,264 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:48,265 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:48,265 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:48,265 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:48,265 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:48,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53998 deadline: 1731208728253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:48,265 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:48,265 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:48,265 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:48,266 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:48,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53984 deadline: 1731208728255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:48,266 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:48,266 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:48,266 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:48,320 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/B/ac5d3b6f752645aaa8e433a0d1b627d0 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/ac5d3b6f752645aaa8e433a0d1b627d0 2024-11-10T03:17:48,324 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c6879c5307a3686e6731f46d96d3ca08/B of c6879c5307a3686e6731f46d96d3ca08 into ac5d3b6f752645aaa8e433a0d1b627d0(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:17:48,324 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c6879c5307a3686e6731f46d96d3ca08: 2024-11-10T03:17:48,324 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., storeName=c6879c5307a3686e6731f46d96d3ca08/B, priority=13, startTime=1731208667841; duration=0sec 2024-11-10T03:17:48,324 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:17:48,324 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c6879c5307a3686e6731f46d96d3ca08:B 2024-11-10T03:17:48,371 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-10T03:17:48,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:48,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. as already flushing 2024-11-10T03:17:48,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:48,371 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:48,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:48,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:48,378 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:48,382 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111021dd711d3c354386b2c5fe57452a943a_c6879c5307a3686e6731f46d96d3ca08 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111021dd711d3c354386b2c5fe57452a943a_c6879c5307a3686e6731f46d96d3ca08 2024-11-10T03:17:48,383 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/A/c3eeab4d778b4e47a44ae13b2bad554f, store: [table=TestAcidGuarantees family=A region=c6879c5307a3686e6731f46d96d3ca08] 2024-11-10T03:17:48,384 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/A/c3eeab4d778b4e47a44ae13b2bad554f is 175, key is test_row_0/A:col10/1731208667128/Put/seqid=0 2024-11-10T03:17:48,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742200_1376 (size=39699) 2024-11-10T03:17:48,524 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-10T03:17:48,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:48,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. as already flushing 2024-11-10T03:17:48,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:48,525 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:48,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:48,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:48,573 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:48,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53924 deadline: 1731208728573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:48,574 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:48,574 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:48,574 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:48,574 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:48,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53984 deadline: 1731208728573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:48,574 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:48,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53998 deadline: 1731208728573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:48,576 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:48,577 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:48,577 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:48,577 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:48,577 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:48,577 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:48,579 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:48,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53958 deadline: 1731208728574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:48,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:48,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:48,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:48,678 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-10T03:17:48,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:48,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. as already flushing 2024-11-10T03:17:48,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:48,679 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:48,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:48,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:48,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-10T03:17:48,801 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=135, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/A/c3eeab4d778b4e47a44ae13b2bad554f 2024-11-10T03:17:48,825 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/B/fbc20d0410d54d96b6547254267e500c is 50, key is test_row_0/B:col10/1731208667128/Put/seqid=0 2024-11-10T03:17:48,834 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-10T03:17:48,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:48,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. as already flushing 2024-11-10T03:17:48,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:48,834 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:48,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:48,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:48,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742201_1377 (size=12101) 2024-11-10T03:17:48,987 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-10T03:17:48,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:48,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. as already flushing 2024-11-10T03:17:48,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:48,987 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:48,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:48,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:49,088 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:49,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53984 deadline: 1731208729083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:49,088 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:49,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53998 deadline: 1731208729083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:49,089 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:49,089 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:49,089 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:49,089 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:49,089 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:49,089 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:49,089 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:49,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53924 deadline: 1731208729083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:49,091 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:49,091 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:49,091 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:49,100 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:49,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53958 deadline: 1731208729093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:49,101 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:49,101 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:49,101 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:49,140 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-10T03:17:49,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:49,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. as already flushing 2024-11-10T03:17:49,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:49,140 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:49,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:49,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:49,269 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/B/fbc20d0410d54d96b6547254267e500c 2024-11-10T03:17:49,287 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/C/893cbf97a9fc4cb0a37a373db52ab29f is 50, key is test_row_0/C:col10/1731208667128/Put/seqid=0 2024-11-10T03:17:49,293 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-10T03:17:49,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:49,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. as already flushing 2024-11-10T03:17:49,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:49,293 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:49,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:49,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:49,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742202_1378 (size=12101) 2024-11-10T03:17:49,331 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/C/893cbf97a9fc4cb0a37a373db52ab29f 2024-11-10T03:17:49,336 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/A/c3eeab4d778b4e47a44ae13b2bad554f as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/c3eeab4d778b4e47a44ae13b2bad554f 2024-11-10T03:17:49,348 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/c3eeab4d778b4e47a44ae13b2bad554f, entries=200, sequenceid=135, filesize=38.8 K 2024-11-10T03:17:49,349 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/B/fbc20d0410d54d96b6547254267e500c as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/fbc20d0410d54d96b6547254267e500c 2024-11-10T03:17:49,353 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/fbc20d0410d54d96b6547254267e500c, entries=150, sequenceid=135, filesize=11.8 K 2024-11-10T03:17:49,353 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/C/893cbf97a9fc4cb0a37a373db52ab29f as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/893cbf97a9fc4cb0a37a373db52ab29f 2024-11-10T03:17:49,361 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/893cbf97a9fc4cb0a37a373db52ab29f, entries=150, sequenceid=135, filesize=11.8 K 2024-11-10T03:17:49,362 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for c6879c5307a3686e6731f46d96d3ca08 in 1505ms, sequenceid=135, compaction requested=false 2024-11-10T03:17:49,362 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c6879c5307a3686e6731f46d96d3ca08: 2024-11-10T03:17:49,450 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-10T03:17:49,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:49,451 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing c6879c5307a3686e6731f46d96d3ca08 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-10T03:17:49,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6879c5307a3686e6731f46d96d3ca08, store=A 2024-11-10T03:17:49,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:17:49,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6879c5307a3686e6731f46d96d3ca08, store=B 2024-11-10T03:17:49,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:17:49,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6879c5307a3686e6731f46d96d3ca08, store=C 2024-11-10T03:17:49,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:17:49,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110fd2ee3e6a34742b18365d885fa819ef7_c6879c5307a3686e6731f46d96d3ca08 is 50, key is test_row_0/A:col10/1731208667913/Put/seqid=0 2024-11-10T03:17:49,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742203_1379 (size=12304) 2024-11-10T03:17:49,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:49,507 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110fd2ee3e6a34742b18365d885fa819ef7_c6879c5307a3686e6731f46d96d3ca08 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110fd2ee3e6a34742b18365d885fa819ef7_c6879c5307a3686e6731f46d96d3ca08 2024-11-10T03:17:49,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/A/7b32c100c9014b25b774f2f43df55a33, store: [table=TestAcidGuarantees family=A region=c6879c5307a3686e6731f46d96d3ca08] 2024-11-10T03:17:49,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/A/7b32c100c9014b25b774f2f43df55a33 is 175, key is test_row_0/A:col10/1731208667913/Put/seqid=0 2024-11-10T03:17:49,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742204_1380 (size=31105) 2024-11-10T03:17:49,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-10T03:17:49,914 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=150, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/A/7b32c100c9014b25b774f2f43df55a33 2024-11-10T03:17:49,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/B/9c00d5e46bf94f218f452829f3cf2ef9 is 50, key is test_row_0/B:col10/1731208667913/Put/seqid=0 2024-11-10T03:17:49,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742205_1381 (size=12151) 2024-11-10T03:17:49,956 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=150 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/B/9c00d5e46bf94f218f452829f3cf2ef9 2024-11-10T03:17:49,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/C/d33f1ad4b2b94214a61cb6e41689b463 is 50, key is test_row_0/C:col10/1731208667913/Put/seqid=0 2024-11-10T03:17:49,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742206_1382 (size=12151) 2024-11-10T03:17:49,984 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=150 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/C/d33f1ad4b2b94214a61cb6e41689b463 2024-11-10T03:17:49,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/A/7b32c100c9014b25b774f2f43df55a33 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/7b32c100c9014b25b774f2f43df55a33 2024-11-10T03:17:49,995 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/7b32c100c9014b25b774f2f43df55a33, entries=150, sequenceid=150, filesize=30.4 K 2024-11-10T03:17:49,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/B/9c00d5e46bf94f218f452829f3cf2ef9 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/9c00d5e46bf94f218f452829f3cf2ef9 2024-11-10T03:17:49,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:49,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:49,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,000 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/9c00d5e46bf94f218f452829f3cf2ef9, entries=150, sequenceid=150, filesize=11.9 K 2024-11-10T03:17:50,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/C/d33f1ad4b2b94214a61cb6e41689b463 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/d33f1ad4b2b94214a61cb6e41689b463 2024-11-10T03:17:50,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,008 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/d33f1ad4b2b94214a61cb6e41689b463, entries=150, sequenceid=150, filesize=11.9 K 2024-11-10T03:17:50,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,010 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=0 B/0 for c6879c5307a3686e6731f46d96d3ca08 in 559ms, sequenceid=150, compaction requested=true 2024-11-10T03:17:50,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for c6879c5307a3686e6731f46d96d3ca08: 2024-11-10T03:17:50,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:50,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-10T03:17:50,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-10T03:17:50,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,016 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-10T03:17:50,016 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4400 sec 2024-11-10T03:17:50,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,019 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 2.4490 sec 2024-11-10T03:17:50,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on c6879c5307a3686e6731f46d96d3ca08 2024-11-10T03:17:50,240 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c6879c5307a3686e6731f46d96d3ca08 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-10T03:17:50,240 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6879c5307a3686e6731f46d96d3ca08, store=A 2024-11-10T03:17:50,240 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:17:50,240 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6879c5307a3686e6731f46d96d3ca08, store=B 2024-11-10T03:17:50,240 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:17:50,240 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6879c5307a3686e6731f46d96d3ca08, store=C 2024-11-10T03:17:50,240 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:17:50,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,248 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411101ce9ed9570eb4ad0a777993a1a2fcf33_c6879c5307a3686e6731f46d96d3ca08 is 50, key is test_row_0/A:col10/1731208670212/Put/seqid=0 2024-11-10T03:17:50,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742207_1383 (size=14794) 2024-11-10T03:17:50,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,260 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,264 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411101ce9ed9570eb4ad0a777993a1a2fcf33_c6879c5307a3686e6731f46d96d3ca08 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411101ce9ed9570eb4ad0a777993a1a2fcf33_c6879c5307a3686e6731f46d96d3ca08 2024-11-10T03:17:50,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,264 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/A/6888815fd6f94a2993960b8ca125b139, store: [table=TestAcidGuarantees family=A region=c6879c5307a3686e6731f46d96d3ca08] 2024-11-10T03:17:50,265 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/A/6888815fd6f94a2993960b8ca125b139 is 175, key is test_row_0/A:col10/1731208670212/Put/seqid=0 2024-11-10T03:17:50,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742208_1384 (size=39749) 2024-11-10T03:17:50,286 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=161, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/A/6888815fd6f94a2993960b8ca125b139 2024-11-10T03:17:50,296 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/B/21db582b735241f4811ea5e2bbb0c5b6 is 50, key is test_row_0/B:col10/1731208670212/Put/seqid=0 2024-11-10T03:17:50,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742209_1385 (size=12151) 2024-11-10T03:17:50,311 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=161 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/B/21db582b735241f4811ea5e2bbb0c5b6 2024-11-10T03:17:50,322 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/C/572b08420f50472dbb97cdf1667c4bb6 is 50, key is test_row_0/C:col10/1731208670212/Put/seqid=0 2024-11-10T03:17:50,335 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:50,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53958 deadline: 1731208730319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:50,335 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:50,335 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:50,335 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:50,338 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731208670298, max timestamp=1731208670333Num uniques 859; 2024-11-10T03:17:50,338 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731208670298, max timestamp=1731208670333Num uniques 859; 2024-11-10T03:17:50,338 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T03:17:50,338 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T03:17:50,338 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731208670298, max timestamp=1731208670333Num uniques 859; 2024-11-10T03:17:50,341 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T03:17:50,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742210_1386 (size=12151) 2024-11-10T03:17:50,343 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=161 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/C/572b08420f50472dbb97cdf1667c4bb6 2024-11-10T03:17:50,348 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/A/6888815fd6f94a2993960b8ca125b139 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/6888815fd6f94a2993960b8ca125b139 2024-11-10T03:17:50,352 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/6888815fd6f94a2993960b8ca125b139, entries=200, sequenceid=161, filesize=38.8 K 2024-11-10T03:17:50,353 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/B/21db582b735241f4811ea5e2bbb0c5b6 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/21db582b735241f4811ea5e2bbb0c5b6 2024-11-10T03:17:50,357 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/21db582b735241f4811ea5e2bbb0c5b6, entries=150, sequenceid=161, filesize=11.9 K 2024-11-10T03:17:50,358 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/C/572b08420f50472dbb97cdf1667c4bb6 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/572b08420f50472dbb97cdf1667c4bb6 2024-11-10T03:17:50,362 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/572b08420f50472dbb97cdf1667c4bb6, entries=150, sequenceid=161, filesize=11.9 K 2024-11-10T03:17:50,363 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=53.67 KB/54960 for c6879c5307a3686e6731f46d96d3ca08 in 124ms, sequenceid=161, compaction requested=true 2024-11-10T03:17:50,363 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c6879c5307a3686e6731f46d96d3ca08: 2024-11-10T03:17:50,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c6879c5307a3686e6731f46d96d3ca08:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T03:17:50,364 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T03:17:50,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:17:50,364 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T03:17:50,365 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141748 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T03:17:50,365 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48644 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T03:17:50,365 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1541): c6879c5307a3686e6731f46d96d3ca08/A is initiating minor compaction (all files) 2024-11-10T03:17:50,365 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1541): c6879c5307a3686e6731f46d96d3ca08/B is initiating minor compaction (all files) 2024-11-10T03:17:50,365 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c6879c5307a3686e6731f46d96d3ca08/B in TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:50,365 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c6879c5307a3686e6731f46d96d3ca08/A in TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:50,365 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/9d701ae590b2482aac0ee6fcdb89bac9, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/c3eeab4d778b4e47a44ae13b2bad554f, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/7b32c100c9014b25b774f2f43df55a33, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/6888815fd6f94a2993960b8ca125b139] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp, totalSize=138.4 K 2024-11-10T03:17:50,365 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/ac5d3b6f752645aaa8e433a0d1b627d0, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/fbc20d0410d54d96b6547254267e500c, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/9c00d5e46bf94f218f452829f3cf2ef9, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/21db582b735241f4811ea5e2bbb0c5b6] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp, totalSize=47.5 K 2024-11-10T03:17:50,365 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:50,365 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. files: [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/9d701ae590b2482aac0ee6fcdb89bac9, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/c3eeab4d778b4e47a44ae13b2bad554f, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/7b32c100c9014b25b774f2f43df55a33, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/6888815fd6f94a2993960b8ca125b139] 2024-11-10T03:17:50,366 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting ac5d3b6f752645aaa8e433a0d1b627d0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731208667003 2024-11-10T03:17:50,366 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9d701ae590b2482aac0ee6fcdb89bac9, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731208667003 2024-11-10T03:17:50,366 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting fbc20d0410d54d96b6547254267e500c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731208667128 2024-11-10T03:17:50,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c6879c5307a3686e6731f46d96d3ca08:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T03:17:50,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:17:50,366 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c00d5e46bf94f218f452829f3cf2ef9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1731208667891 2024-11-10T03:17:50,366 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting c3eeab4d778b4e47a44ae13b2bad554f, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731208667128 2024-11-10T03:17:50,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c6879c5307a3686e6731f46d96d3ca08:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T03:17:50,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T03:17:50,367 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 21db582b735241f4811ea5e2bbb0c5b6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1731208670157 2024-11-10T03:17:50,367 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b32c100c9014b25b774f2f43df55a33, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1731208667891 2024-11-10T03:17:50,367 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6888815fd6f94a2993960b8ca125b139, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1731208670157 2024-11-10T03:17:50,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on c6879c5307a3686e6731f46d96d3ca08 2024-11-10T03:17:50,376 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c6879c5307a3686e6731f46d96d3ca08 3/3 column families, dataSize=60.38 KB heapSize=161.16 KB 2024-11-10T03:17:50,379 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c6879c5307a3686e6731f46d96d3ca08] 2024-11-10T03:17:50,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6879c5307a3686e6731f46d96d3ca08, store=A 2024-11-10T03:17:50,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T03:17:50,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6879c5307a3686e6731f46d96d3ca08, store=B 2024-11-10T03:17:50,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T03:17:50,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6879c5307a3686e6731f46d96d3ca08, store=C 2024-11-10T03:17:50,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T03:17:50,387 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c6879c5307a3686e6731f46d96d3ca08#B#compaction#325 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:17:50,387 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/B/6570da8fc3ce4670ad83f27ff6ecb01d is 50, key is test_row_0/B:col10/1731208670212/Put/seqid=0 2024-11-10T03:17:50,393 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411106ec91dfb3ea5421aa12e16f7c1f2bfa9_c6879c5307a3686e6731f46d96d3ca08 store=[table=TestAcidGuarantees family=A region=c6879c5307a3686e6731f46d96d3ca08] 2024-11-10T03:17:50,395 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411106ec91dfb3ea5421aa12e16f7c1f2bfa9_c6879c5307a3686e6731f46d96d3ca08, store=[table=TestAcidGuarantees family=A region=c6879c5307a3686e6731f46d96d3ca08] 2024-11-10T03:17:50,396 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411106ec91dfb3ea5421aa12e16f7c1f2bfa9_c6879c5307a3686e6731f46d96d3ca08 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c6879c5307a3686e6731f46d96d3ca08] 2024-11-10T03:17:50,418 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411106b940f72035c4c2a864bffc2a76c46ab_c6879c5307a3686e6731f46d96d3ca08 is 50, key is test_row_0/A:col10/1731208670316/Put/seqid=0 2024-11-10T03:17:50,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742211_1387 (size=12527) 2024-11-10T03:17:50,455 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/B/6570da8fc3ce4670ad83f27ff6ecb01d as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/6570da8fc3ce4670ad83f27ff6ecb01d 2024-11-10T03:17:50,461 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c6879c5307a3686e6731f46d96d3ca08/B of c6879c5307a3686e6731f46d96d3ca08 into 6570da8fc3ce4670ad83f27ff6ecb01d(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:17:50,462 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c6879c5307a3686e6731f46d96d3ca08: 2024-11-10T03:17:50,462 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., storeName=c6879c5307a3686e6731f46d96d3ca08/B, priority=12, startTime=1731208670364; duration=0sec 2024-11-10T03:17:50,462 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T03:17:50,462 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c6879c5307a3686e6731f46d96d3ca08:B 2024-11-10T03:17:50,462 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T03:17:50,465 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48644 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T03:17:50,465 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1541): c6879c5307a3686e6731f46d96d3ca08/C is initiating minor compaction (all files) 2024-11-10T03:17:50,465 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c6879c5307a3686e6731f46d96d3ca08/C in TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:50,465 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/6a2ee4fda36f443ea25a79724d3b17bd, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/893cbf97a9fc4cb0a37a373db52ab29f, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/d33f1ad4b2b94214a61cb6e41689b463, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/572b08420f50472dbb97cdf1667c4bb6] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp, totalSize=47.5 K 2024-11-10T03:17:50,466 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a2ee4fda36f443ea25a79724d3b17bd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731208667003 2024-11-10T03:17:50,466 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 893cbf97a9fc4cb0a37a373db52ab29f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731208667128 2024-11-10T03:17:50,466 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting d33f1ad4b2b94214a61cb6e41689b463, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1731208667891 2024-11-10T03:17:50,466 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 572b08420f50472dbb97cdf1667c4bb6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1731208670157 2024-11-10T03:17:50,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742212_1388 (size=4469) 2024-11-10T03:17:50,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742213_1389 (size=14794) 2024-11-10T03:17:50,491 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,491 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:50,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53924 deadline: 1731208730471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:50,492 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:50,492 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:50,492 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:50,493 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c6879c5307a3686e6731f46d96d3ca08#C#compaction#327 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:17:50,493 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731208670427, max timestamp=1731208670489Num uniques 859; 2024-11-10T03:17:50,493 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731208670427, max timestamp=1731208670489Num uniques 859; 2024-11-10T03:17:50,493 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/C/bfb14d1aafa24f96973327c4d2720ef1 is 50, key is test_row_0/C:col10/1731208670212/Put/seqid=0 2024-11-10T03:17:50,493 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T03:17:50,493 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T03:17:50,497 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731208670427, max timestamp=1731208670489Num uniques 859; 2024-11-10T03:17:50,497 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T03:17:50,499 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411106b940f72035c4c2a864bffc2a76c46ab_c6879c5307a3686e6731f46d96d3ca08 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411106b940f72035c4c2a864bffc2a76c46ab_c6879c5307a3686e6731f46d96d3ca08 2024-11-10T03:17:50,502 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/A/151470833e7c4978a450830a8a879aa5, store: [table=TestAcidGuarantees family=A region=c6879c5307a3686e6731f46d96d3ca08] 2024-11-10T03:17:50,503 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/A/151470833e7c4978a450830a8a879aa5 is 175, key is test_row_0/A:col10/1731208670316/Put/seqid=0 2024-11-10T03:17:50,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742214_1390 (size=12527) 2024-11-10T03:17:50,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742215_1391 (size=39749) 2024-11-10T03:17:50,524 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/C/bfb14d1aafa24f96973327c4d2720ef1 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/bfb14d1aafa24f96973327c4d2720ef1 2024-11-10T03:17:50,524 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=187, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/A/151470833e7c4978a450830a8a879aa5 2024-11-10T03:17:50,549 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c6879c5307a3686e6731f46d96d3ca08/C of c6879c5307a3686e6731f46d96d3ca08 into bfb14d1aafa24f96973327c4d2720ef1(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:17:50,549 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c6879c5307a3686e6731f46d96d3ca08: 2024-11-10T03:17:50,549 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., storeName=c6879c5307a3686e6731f46d96d3ca08/C, priority=12, startTime=1731208670367; duration=0sec 2024-11-10T03:17:50,549 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:17:50,549 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c6879c5307a3686e6731f46d96d3ca08:C 2024-11-10T03:17:50,557 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/B/d116045cd9144320975e85b8ad507879 is 50, key is test_row_0/B:col10/1731208670316/Put/seqid=0 2024-11-10T03:17:50,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742216_1392 (size=12151) 2024-11-10T03:17:50,569 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=187 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/B/d116045cd9144320975e85b8ad507879 2024-11-10T03:17:50,584 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/C/ece0b4c6165448e990025df7b77e00ad is 50, key is test_row_0/C:col10/1731208670316/Put/seqid=0 2024-11-10T03:17:50,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742217_1393 (size=12151) 2024-11-10T03:17:50,610 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=187 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/C/ece0b4c6165448e990025df7b77e00ad 2024-11-10T03:17:50,614 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/A/151470833e7c4978a450830a8a879aa5 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/151470833e7c4978a450830a8a879aa5 2024-11-10T03:17:50,618 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/151470833e7c4978a450830a8a879aa5, entries=200, sequenceid=187, filesize=38.8 K 2024-11-10T03:17:50,619 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/B/d116045cd9144320975e85b8ad507879 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/d116045cd9144320975e85b8ad507879 2024-11-10T03:17:50,622 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/d116045cd9144320975e85b8ad507879, entries=150, sequenceid=187, filesize=11.9 K 2024-11-10T03:17:50,623 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/C/ece0b4c6165448e990025df7b77e00ad as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/ece0b4c6165448e990025df7b77e00ad 2024-11-10T03:17:50,627 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/ece0b4c6165448e990025df7b77e00ad, entries=150, sequenceid=187, filesize=11.9 K 2024-11-10T03:17:50,635 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~181.03 KB/185376, currentSize=107.34 KB/109920 for c6879c5307a3686e6731f46d96d3ca08 in 260ms, sequenceid=187, compaction requested=false 2024-11-10T03:17:50,635 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c6879c5307a3686e6731f46d96d3ca08: 2024-11-10T03:17:50,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on c6879c5307a3686e6731f46d96d3ca08 2024-11-10T03:17:50,636 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c6879c5307a3686e6731f46d96d3ca08 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-10T03:17:50,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6879c5307a3686e6731f46d96d3ca08, store=A 2024-11-10T03:17:50,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T03:17:50,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6879c5307a3686e6731f46d96d3ca08, store=B 2024-11-10T03:17:50,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T03:17:50,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6879c5307a3686e6731f46d96d3ca08, store=C 2024-11-10T03:17:50,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T03:17:50,648 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411103634af3a490f4565b90d69dae76cacfd_c6879c5307a3686e6731f46d96d3ca08 is 50, key is test_row_0/A:col10/1731208670544/Put/seqid=0 2024-11-10T03:17:50,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742218_1394 (size=12304) 2024-11-10T03:17:50,656 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:50,660 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411103634af3a490f4565b90d69dae76cacfd_c6879c5307a3686e6731f46d96d3ca08 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411103634af3a490f4565b90d69dae76cacfd_c6879c5307a3686e6731f46d96d3ca08 2024-11-10T03:17:50,661 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/A/195069d59f334396bc408f1c793cebbf, store: [table=TestAcidGuarantees family=A region=c6879c5307a3686e6731f46d96d3ca08] 2024-11-10T03:17:50,662 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/A/195069d59f334396bc408f1c793cebbf is 175, key is test_row_0/A:col10/1731208670544/Put/seqid=0 2024-11-10T03:17:50,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742219_1395 (size=31105) 2024-11-10T03:17:50,682 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:50,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:50,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53924 deadline: 1731208730661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:50,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53958 deadline: 1731208730668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:50,683 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:50,683 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:50,683 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:50,683 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:50,683 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:50,683 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:50,700 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:50,701 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:50,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53984 deadline: 1731208730682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:50,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53998 deadline: 1731208730683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:50,701 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:50,701 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:50,701 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:50,703 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:50,704 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:50,704 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:50,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:50,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53958 deadline: 1731208730793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:50,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:50,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53924 deadline: 1731208730793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:50,796 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:50,796 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:50,796 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:50,796 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:50,796 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:50,796 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:50,817 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:50,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53998 deadline: 1731208730803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:50,820 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:50,820 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:50,820 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:50,829 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:50,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53984 deadline: 1731208730813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:50,829 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:50,829 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:50,829 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:50,881 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c6879c5307a3686e6731f46d96d3ca08#A#compaction#324 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:17:50,882 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/A/3d18f839f37a4319b454f86617be30b5 is 175, key is test_row_0/A:col10/1731208670212/Put/seqid=0 2024-11-10T03:17:50,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742220_1396 (size=31481) 2024-11-10T03:17:51,026 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:51,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53924 deadline: 1731208731003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:51,026 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:51,026 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:51,026 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:51,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53958 deadline: 1731208731011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:51,026 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:51,027 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:51,027 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:51,027 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:51,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:51,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53998 deadline: 1731208731023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:51,033 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:51,033 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:51,033 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:51,041 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:51,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53984 deadline: 1731208731033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:51,043 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:51,043 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:51,043 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:51,095 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=224, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/A/195069d59f334396bc408f1c793cebbf 2024-11-10T03:17:51,118 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/B/1f5c6b9ee4e544509063b6f91c65bc59 is 50, key is test_row_0/B:col10/1731208670544/Put/seqid=0 2024-11-10T03:17:51,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742221_1397 (size=12151) 2024-11-10T03:17:51,152 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/B/1f5c6b9ee4e544509063b6f91c65bc59 2024-11-10T03:17:51,163 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/C/a401c90232b841e8badc606f7378f0bb is 50, key is test_row_0/C:col10/1731208670544/Put/seqid=0 2024-11-10T03:17:51,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742222_1398 (size=12151) 2024-11-10T03:17:51,183 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/C/a401c90232b841e8badc606f7378f0bb 2024-11-10T03:17:51,187 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/A/195069d59f334396bc408f1c793cebbf as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/195069d59f334396bc408f1c793cebbf 2024-11-10T03:17:51,192 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/195069d59f334396bc408f1c793cebbf, entries=150, sequenceid=224, filesize=30.4 K 2024-11-10T03:17:51,192 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/B/1f5c6b9ee4e544509063b6f91c65bc59 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/1f5c6b9ee4e544509063b6f91c65bc59 2024-11-10T03:17:51,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,200 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/1f5c6b9ee4e544509063b6f91c65bc59, entries=150, sequenceid=224, filesize=11.9 K 2024-11-10T03:17:51,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,202 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/C/a401c90232b841e8badc606f7378f0bb as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/a401c90232b841e8badc606f7378f0bb 2024-11-10T03:17:51,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,208 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/a401c90232b841e8badc606f7378f0bb, entries=150, sequenceid=224, filesize=11.9 K 2024-11-10T03:17:51,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,209 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for c6879c5307a3686e6731f46d96d3ca08 in 573ms, sequenceid=224, compaction requested=true 2024-11-10T03:17:51,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,209 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c6879c5307a3686e6731f46d96d3ca08: 2024-11-10T03:17:51,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,209 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 4 compacting, 2 eligible, 16 blocking 2024-11-10T03:17:51,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c6879c5307a3686e6731f46d96d3ca08:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T03:17:51,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:17:51,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c6879c5307a3686e6731f46d96d3ca08:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T03:17:51,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T03:17:51,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c6879c5307a3686e6731f46d96d3ca08:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T03:17:51,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-10T03:17:51,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,210 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-10T03:17:51,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,210 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-10T03:17:51,210 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. because compaction request was cancelled 2024-11-10T03:17:51,210 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c6879c5307a3686e6731f46d96d3ca08:A 2024-11-10T03:17:51,211 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T03:17:51,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,214 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T03:17:51,214 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1541): c6879c5307a3686e6731f46d96d3ca08/B is initiating minor compaction (all files) 2024-11-10T03:17:51,214 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c6879c5307a3686e6731f46d96d3ca08/B in TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:51,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,214 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/6570da8fc3ce4670ad83f27ff6ecb01d, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/d116045cd9144320975e85b8ad507879, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/1f5c6b9ee4e544509063b6f91c65bc59] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp, totalSize=36.0 K 2024-11-10T03:17:51,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,215 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 6570da8fc3ce4670ad83f27ff6ecb01d, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1731208670157 2024-11-10T03:17:51,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,216 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting d116045cd9144320975e85b8ad507879, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1731208670316 2024-11-10T03:17:51,217 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f5c6b9ee4e544509063b6f91c65bc59, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1731208670544 2024-11-10T03:17:51,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,258 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c6879c5307a3686e6731f46d96d3ca08#B#compaction#333 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:17:51,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,258 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/B/8e38bf8df49a4e3da25345fb2993fa16 is 50, key is test_row_0/B:col10/1731208670544/Put/seqid=0 2024-11-10T03:17:51,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742223_1399 (size=12629) 2024-11-10T03:17:51,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,307 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/B/8e38bf8df49a4e3da25345fb2993fa16 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/8e38bf8df49a4e3da25345fb2993fa16 2024-11-10T03:17:51,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,314 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c6879c5307a3686e6731f46d96d3ca08/B of c6879c5307a3686e6731f46d96d3ca08 into 8e38bf8df49a4e3da25345fb2993fa16(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:17:51,314 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c6879c5307a3686e6731f46d96d3ca08: 2024-11-10T03:17:51,314 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., storeName=c6879c5307a3686e6731f46d96d3ca08/B, priority=13, startTime=1731208671209; duration=0sec 2024-11-10T03:17:51,314 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T03:17:51,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,314 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c6879c5307a3686e6731f46d96d3ca08:B 2024-11-10T03:17:51,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,314 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T03:17:51,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,319 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T03:17:51,319 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1541): c6879c5307a3686e6731f46d96d3ca08/C is initiating minor compaction (all files) 2024-11-10T03:17:51,319 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c6879c5307a3686e6731f46d96d3ca08/C in TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:51,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,319 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/bfb14d1aafa24f96973327c4d2720ef1, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/ece0b4c6165448e990025df7b77e00ad, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/a401c90232b841e8badc606f7378f0bb] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp, totalSize=36.0 K 2024-11-10T03:17:51,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,320 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting bfb14d1aafa24f96973327c4d2720ef1, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1731208670157 2024-11-10T03:17:51,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,320 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting ece0b4c6165448e990025df7b77e00ad, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1731208670316 2024-11-10T03:17:51,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,321 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting a401c90232b841e8badc606f7378f0bb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1731208670544 2024-11-10T03:17:51,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,330 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c6879c5307a3686e6731f46d96d3ca08#C#compaction#334 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:17:51,331 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/C/fdfeda849bc74df286979cf33924ef3b is 50, key is test_row_0/C:col10/1731208670544/Put/seqid=0 2024-11-10T03:17:51,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,338 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/A/3d18f839f37a4319b454f86617be30b5 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/3d18f839f37a4319b454f86617be30b5 2024-11-10T03:17:51,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,344 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c6879c5307a3686e6731f46d96d3ca08/A of c6879c5307a3686e6731f46d96d3ca08 into 3d18f839f37a4319b454f86617be30b5(size=30.7 K), total size for store is 99.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:17:51,344 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c6879c5307a3686e6731f46d96d3ca08: 2024-11-10T03:17:51,344 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., storeName=c6879c5307a3686e6731f46d96d3ca08/A, priority=12, startTime=1731208670363; duration=0sec 2024-11-10T03:17:51,344 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:17:51,344 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c6879c5307a3686e6731f46d96d3ca08:A 2024-11-10T03:17:51,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742224_1400 (size=12629) 2024-11-10T03:17:51,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,362 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c6879c5307a3686e6731f46d96d3ca08 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-10T03:17:51,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6879c5307a3686e6731f46d96d3ca08, store=A 2024-11-10T03:17:51,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:17:51,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on c6879c5307a3686e6731f46d96d3ca08 2024-11-10T03:17:51,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6879c5307a3686e6731f46d96d3ca08, store=B 2024-11-10T03:17:51,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:17:51,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6879c5307a3686e6731f46d96d3ca08, store=C 2024-11-10T03:17:51,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:17:51,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,378 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411100708b998fe7f40aa8355593a38c810ea_c6879c5307a3686e6731f46d96d3ca08 is 50, key is test_row_1/A:col10/1731208670655/Put/seqid=0 2024-11-10T03:17:51,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742225_1401 (size=12304) 2024-11-10T03:17:51,485 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:51,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53924 deadline: 1731208731465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:51,486 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:51,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:51,486 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:51,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53984 deadline: 1731208731466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:51,486 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:51,486 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:51,486 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:51,486 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:51,494 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:51,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53958 deadline: 1731208731476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:51,494 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:51,494 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:51,494 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:51,502 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:51,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53998 deadline: 1731208731486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:51,503 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:51,503 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:51,503 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:51,559 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:51,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54002 deadline: 1731208731553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:51,560 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 394d2222f724,38139,1731208572054 for 'test_row_0' in c6879c5307a3686e6731f46d96d3ca08 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8270 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T03:17:51,560 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:51,560 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:51,561 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:51,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:51,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53984 deadline: 1731208731593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:51,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:51,601 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:51,601 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:51,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53924 deadline: 1731208731593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:51,601 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:51,601 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:51,602 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:51,602 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:51,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:51,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53958 deadline: 1731208731603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:51,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:51,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:51,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:51,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:51,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53998 deadline: 1731208731613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:51,623 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:51,623 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:51,623 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:51,757 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/C/fdfeda849bc74df286979cf33924ef3b as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/fdfeda849bc74df286979cf33924ef3b 2024-11-10T03:17:51,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-10T03:17:51,761 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T03:17:51,762 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T03:17:51,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-10T03:17:51,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-10T03:17:51,764 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T03:17:51,764 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T03:17:51,764 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T03:17:51,768 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c6879c5307a3686e6731f46d96d3ca08/C of c6879c5307a3686e6731f46d96d3ca08 into fdfeda849bc74df286979cf33924ef3b(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:17:51,769 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c6879c5307a3686e6731f46d96d3ca08: 2024-11-10T03:17:51,769 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., storeName=c6879c5307a3686e6731f46d96d3ca08/C, priority=13, startTime=1731208671209; duration=0sec 2024-11-10T03:17:51,769 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:17:51,769 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c6879c5307a3686e6731f46d96d3ca08:C 2024-11-10T03:17:51,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:51,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53984 deadline: 1731208731803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:51,814 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:51,814 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:51,814 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:51,816 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:51,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53924 deadline: 1731208731815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:51,817 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:51,817 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:51,817 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:51,817 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:51,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53958 deadline: 1731208731815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:51,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:51,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:51,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:51,819 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:51,825 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411100708b998fe7f40aa8355593a38c810ea_c6879c5307a3686e6731f46d96d3ca08 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411100708b998fe7f40aa8355593a38c810ea_c6879c5307a3686e6731f46d96d3ca08 2024-11-10T03:17:51,826 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/A/1405fa84825d4ceeb14587e42be94948, store: [table=TestAcidGuarantees family=A region=c6879c5307a3686e6731f46d96d3ca08] 2024-11-10T03:17:51,827 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/A/1405fa84825d4ceeb14587e42be94948 is 175, key is test_row_1/A:col10/1731208670655/Put/seqid=0 2024-11-10T03:17:51,841 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:51,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53998 deadline: 1731208731833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:51,843 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:51,843 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:51,843 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:51,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-10T03:17:51,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742226_1402 (size=31101) 2024-11-10T03:17:51,917 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-10T03:17:51,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:51,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. as already flushing 2024-11-10T03:17:51,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:51,918 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:51,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:51,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:52,072 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-10T03:17:52,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:52,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. as already flushing 2024-11-10T03:17:52,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:52,073 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:52,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:52,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:52,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-10T03:17:52,129 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:52,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53924 deadline: 1731208732123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:52,129 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:52,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53984 deadline: 1731208732123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:52,130 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:52,130 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:52,130 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:52,130 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:52,130 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:52,130 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:52,130 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:52,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53958 deadline: 1731208732123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:52,131 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:52,131 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:52,131 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:52,158 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:52,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53998 deadline: 1731208732153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:52,159 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:52,160 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:52,160 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:52,231 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-10T03:17:52,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:52,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. as already flushing 2024-11-10T03:17:52,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:52,232 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:52,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:52,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:52,290 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=238, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/A/1405fa84825d4ceeb14587e42be94948 2024-11-10T03:17:52,300 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/B/cf91029945ef4b14bf41dc548b555934 is 50, key is test_row_1/B:col10/1731208670655/Put/seqid=0 2024-11-10T03:17:52,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742227_1403 (size=9757) 2024-11-10T03:17:52,310 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/B/cf91029945ef4b14bf41dc548b555934 2024-11-10T03:17:52,328 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/C/ce67747827d54eb8bbc394449199d798 is 50, key is test_row_1/C:col10/1731208670655/Put/seqid=0 2024-11-10T03:17:52,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742228_1404 (size=9757) 2024-11-10T03:17:52,370 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/C/ce67747827d54eb8bbc394449199d798 2024-11-10T03:17:52,380 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/A/1405fa84825d4ceeb14587e42be94948 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/1405fa84825d4ceeb14587e42be94948 2024-11-10T03:17:52,383 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/1405fa84825d4ceeb14587e42be94948, entries=150, sequenceid=238, filesize=30.4 K 2024-11-10T03:17:52,384 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/B/cf91029945ef4b14bf41dc548b555934 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/cf91029945ef4b14bf41dc548b555934 2024-11-10T03:17:52,386 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-10T03:17:52,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:52,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. as already flushing 2024-11-10T03:17:52,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:52,386 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:52,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:52,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:52,392 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/cf91029945ef4b14bf41dc548b555934, entries=100, sequenceid=238, filesize=9.5 K 2024-11-10T03:17:52,393 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/C/ce67747827d54eb8bbc394449199d798 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/ce67747827d54eb8bbc394449199d798 2024-11-10T03:17:52,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-10T03:17:52,398 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/ce67747827d54eb8bbc394449199d798, entries=100, sequenceid=238, filesize=9.5 K 2024-11-10T03:17:52,399 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for c6879c5307a3686e6731f46d96d3ca08 in 1037ms, sequenceid=238, compaction requested=true 2024-11-10T03:17:52,399 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c6879c5307a3686e6731f46d96d3ca08: 2024-11-10T03:17:52,399 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T03:17:52,401 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c6879c5307a3686e6731f46d96d3ca08:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T03:17:52,401 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:17:52,401 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-10T03:17:52,401 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c6879c5307a3686e6731f46d96d3ca08:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T03:17:52,401 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:17:52,401 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c6879c5307a3686e6731f46d96d3ca08:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T03:17:52,401 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T03:17:52,403 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133436 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T03:17:52,403 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1541): c6879c5307a3686e6731f46d96d3ca08/A is initiating minor compaction (all files) 2024-11-10T03:17:52,403 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c6879c5307a3686e6731f46d96d3ca08/A in TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:52,403 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/3d18f839f37a4319b454f86617be30b5, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/151470833e7c4978a450830a8a879aa5, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/195069d59f334396bc408f1c793cebbf, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/1405fa84825d4ceeb14587e42be94948] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp, totalSize=130.3 K 2024-11-10T03:17:52,403 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:52,404 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. files: [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/3d18f839f37a4319b454f86617be30b5, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/151470833e7c4978a450830a8a879aa5, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/195069d59f334396bc408f1c793cebbf, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/1405fa84825d4ceeb14587e42be94948] 2024-11-10T03:17:52,404 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-10T03:17:52,404 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-10T03:17:52,404 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. because compaction request was cancelled 2024-11-10T03:17:52,404 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c6879c5307a3686e6731f46d96d3ca08:B 2024-11-10T03:17:52,404 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-10T03:17:52,404 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d18f839f37a4319b454f86617be30b5, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1731208670157 2024-11-10T03:17:52,405 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 151470833e7c4978a450830a8a879aa5, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1731208670316 2024-11-10T03:17:52,405 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-10T03:17:52,405 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-10T03:17:52,405 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 195069d59f334396bc408f1c793cebbf, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1731208670544 2024-11-10T03:17:52,405 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. because compaction request was cancelled 2024-11-10T03:17:52,405 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c6879c5307a3686e6731f46d96d3ca08:C 2024-11-10T03:17:52,405 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1405fa84825d4ceeb14587e42be94948, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731208670655 2024-11-10T03:17:52,429 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c6879c5307a3686e6731f46d96d3ca08] 2024-11-10T03:17:52,437 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411106d9d2b6770514fcf88272edb51616b0f_c6879c5307a3686e6731f46d96d3ca08 store=[table=TestAcidGuarantees family=A region=c6879c5307a3686e6731f46d96d3ca08] 2024-11-10T03:17:52,439 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411106d9d2b6770514fcf88272edb51616b0f_c6879c5307a3686e6731f46d96d3ca08, store=[table=TestAcidGuarantees family=A region=c6879c5307a3686e6731f46d96d3ca08] 2024-11-10T03:17:52,439 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411106d9d2b6770514fcf88272edb51616b0f_c6879c5307a3686e6731f46d96d3ca08 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c6879c5307a3686e6731f46d96d3ca08] 2024-11-10T03:17:52,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742229_1405 (size=4469) 2024-11-10T03:17:52,541 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-10T03:17:52,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:52,541 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing c6879c5307a3686e6731f46d96d3ca08 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-10T03:17:52,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6879c5307a3686e6731f46d96d3ca08, store=A 2024-11-10T03:17:52,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:17:52,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6879c5307a3686e6731f46d96d3ca08, store=B 2024-11-10T03:17:52,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:17:52,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6879c5307a3686e6731f46d96d3ca08, store=C 2024-11-10T03:17:52,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:17:52,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110f8e8929951cd4505acdff51acd208f8d_c6879c5307a3686e6731f46d96d3ca08 is 50, key is test_row_0/A:col10/1731208671472/Put/seqid=0 2024-11-10T03:17:52,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742230_1406 (size=12354) 2024-11-10T03:17:52,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:52,610 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110f8e8929951cd4505acdff51acd208f8d_c6879c5307a3686e6731f46d96d3ca08 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110f8e8929951cd4505acdff51acd208f8d_c6879c5307a3686e6731f46d96d3ca08 2024-11-10T03:17:52,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/A/87d156ff19d44e99b2ee2df8e93ccaa3, store: [table=TestAcidGuarantees family=A region=c6879c5307a3686e6731f46d96d3ca08] 2024-11-10T03:17:52,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/A/87d156ff19d44e99b2ee2df8e93ccaa3 is 175, key is test_row_0/A:col10/1731208671472/Put/seqid=0 2024-11-10T03:17:52,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742231_1407 (size=31155) 2024-11-10T03:17:52,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on c6879c5307a3686e6731f46d96d3ca08 2024-11-10T03:17:52,644 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. as already flushing 2024-11-10T03:17:52,687 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:52,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53924 deadline: 1731208732673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:52,687 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:52,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53998 deadline: 1731208732675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:52,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:52,688 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:52,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53984 deadline: 1731208732682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:52,688 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:52,688 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:52,692 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:52,692 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:52,692 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:52,692 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:52,692 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:52,692 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:52,694 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:52,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53958 deadline: 1731208732684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:52,696 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:52,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:52,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:52,808 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:52,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53984 deadline: 1731208732803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:52,808 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:52,808 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:52,808 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:52,808 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:52,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53924 deadline: 1731208732803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:52,809 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:52,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53958 deadline: 1731208732803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:52,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:52,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:52,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:52,810 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:52,810 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:52,810 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:52,861 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c6879c5307a3686e6731f46d96d3ca08#A#compaction#338 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:17:52,862 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/A/1129c49e7cd4436684186c32375c56d0 is 175, key is test_row_0/A:col10/1731208670544/Put/seqid=0 2024-11-10T03:17:52,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742232_1408 (size=31724) 2024-11-10T03:17:52,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-10T03:17:53,022 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=259, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/A/87d156ff19d44e99b2ee2df8e93ccaa3 2024-11-10T03:17:53,024 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:53,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53958 deadline: 1731208733013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:53,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:53,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:53,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:53,025 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:53,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53984 deadline: 1731208733013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:53,025 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:53,025 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:53,025 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:53,025 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:53,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53924 deadline: 1731208733013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:53,026 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:53,026 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:53,026 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:53,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/B/cfccbadf71214c25ba968465be75bffe is 50, key is test_row_0/B:col10/1731208671472/Put/seqid=0 2024-11-10T03:17:53,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742233_1409 (size=12201) 2024-11-10T03:17:53,298 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/A/1129c49e7cd4436684186c32375c56d0 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/1129c49e7cd4436684186c32375c56d0 2024-11-10T03:17:53,302 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c6879c5307a3686e6731f46d96d3ca08/A of c6879c5307a3686e6731f46d96d3ca08 into 1129c49e7cd4436684186c32375c56d0(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:17:53,302 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c6879c5307a3686e6731f46d96d3ca08: 2024-11-10T03:17:53,302 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., storeName=c6879c5307a3686e6731f46d96d3ca08/A, priority=12, startTime=1731208672399; duration=0sec 2024-11-10T03:17:53,302 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:17:53,302 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c6879c5307a3686e6731f46d96d3ca08:A 2024-11-10T03:17:53,340 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:53,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53958 deadline: 1731208733333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:53,340 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:53,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53984 deadline: 1731208733333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:53,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:53,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:53,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:53,340 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:53,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53924 deadline: 1731208733333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:53,340 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:53,340 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:53,340 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:53,341 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:53,341 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:53,341 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:53,461 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/B/cfccbadf71214c25ba968465be75bffe 2024-11-10T03:17:53,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/C/4a9152e5371d4d0fb00ec7bbf1a332a7 is 50, key is test_row_0/C:col10/1731208671472/Put/seqid=0 2024-11-10T03:17:53,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742234_1410 (size=12201) 2024-11-10T03:17:53,701 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:53,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53998 deadline: 1731208733693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:53,702 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 394d2222f724,38139,1731208572054 for 'test_row_2' in c6879c5307a3686e6731f46d96d3ca08 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2216 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T03:17:53,702 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:53,702 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:53,702 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:53,863 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:53,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53958 deadline: 1731208733846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:53,868 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:53,868 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:53,868 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:53,868 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:53,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53924 deadline: 1731208733846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:53,869 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:53,869 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:53,869 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:53,869 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:53,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53984 deadline: 1731208733856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:53,869 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:53,869 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:53,869 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:53,895 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/C/4a9152e5371d4d0fb00ec7bbf1a332a7 2024-11-10T03:17:53,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/A/87d156ff19d44e99b2ee2df8e93ccaa3 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/87d156ff19d44e99b2ee2df8e93ccaa3 2024-11-10T03:17:53,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-10T03:17:53,949 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/87d156ff19d44e99b2ee2df8e93ccaa3, entries=150, sequenceid=259, filesize=30.4 K 2024-11-10T03:17:53,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/B/cfccbadf71214c25ba968465be75bffe as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/cfccbadf71214c25ba968465be75bffe 2024-11-10T03:17:53,967 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/cfccbadf71214c25ba968465be75bffe, entries=150, sequenceid=259, filesize=11.9 K 2024-11-10T03:17:53,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/C/4a9152e5371d4d0fb00ec7bbf1a332a7 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/4a9152e5371d4d0fb00ec7bbf1a332a7 2024-11-10T03:17:53,994 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/4a9152e5371d4d0fb00ec7bbf1a332a7, entries=150, sequenceid=259, filesize=11.9 K 2024-11-10T03:17:53,995 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for c6879c5307a3686e6731f46d96d3ca08 in 1454ms, sequenceid=259, compaction requested=true 2024-11-10T03:17:53,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for c6879c5307a3686e6731f46d96d3ca08: 2024-11-10T03:17:53,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:53,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-10T03:17:53,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-10T03:17:53,999 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-10T03:17:53,999 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2330 sec 2024-11-10T03:17:54,002 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 2.2390 sec 2024-11-10T03:17:54,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on c6879c5307a3686e6731f46d96d3ca08 2024-11-10T03:17:54,884 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c6879c5307a3686e6731f46d96d3ca08 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-10T03:17:54,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6879c5307a3686e6731f46d96d3ca08, store=A 2024-11-10T03:17:54,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:17:54,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6879c5307a3686e6731f46d96d3ca08, store=B 2024-11-10T03:17:54,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:17:54,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6879c5307a3686e6731f46d96d3ca08, store=C 2024-11-10T03:17:54,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:17:54,895 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111039030f88612d4fa8bb4df8957fd173cf_c6879c5307a3686e6731f46d96d3ca08 is 50, key is test_row_0/A:col10/1731208674880/Put/seqid=0 2024-11-10T03:17:54,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742235_1411 (size=14994) 2024-11-10T03:17:54,971 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:54,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53984 deadline: 1731208734959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:54,972 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:54,972 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:54,972 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:54,982 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:54,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53958 deadline: 1731208734966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:54,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:54,982 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:54,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:54,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:54,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53924 deadline: 1731208734966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:54,983 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:54,983 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:54,983 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:55,085 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:55,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53984 deadline: 1731208735073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:55,085 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:55,086 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:55,086 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:55,102 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:55,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53958 deadline: 1731208735093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:55,102 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:55,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53924 deadline: 1731208735093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:55,103 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:55,103 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:55,103 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:55,104 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:55,104 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:55,104 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:55,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:55,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53984 deadline: 1731208735292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:55,304 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:55,304 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:55,304 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:55,314 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:55,317 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111039030f88612d4fa8bb4df8957fd173cf_c6879c5307a3686e6731f46d96d3ca08 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111039030f88612d4fa8bb4df8957fd173cf_c6879c5307a3686e6731f46d96d3ca08 2024-11-10T03:17:55,318 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/A/9afd9eff4e5d4caf8726330b3c2af5db, store: [table=TestAcidGuarantees family=A region=c6879c5307a3686e6731f46d96d3ca08] 2024-11-10T03:17:55,319 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/A/9afd9eff4e5d4caf8726330b3c2af5db is 175, key is test_row_0/A:col10/1731208674880/Put/seqid=0 2024-11-10T03:17:55,321 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:55,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53924 deadline: 1731208735313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:55,322 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:55,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53958 deadline: 1731208735313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:55,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742236_1412 (size=39949) 2024-11-10T03:17:55,322 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:55,322 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:55,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:55,324 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:55,324 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:55,324 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:55,325 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=273, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/A/9afd9eff4e5d4caf8726330b3c2af5db 2024-11-10T03:17:55,336 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/B/22ad3ae2088745658a17fc0386b650ec is 50, key is test_row_0/B:col10/1731208674880/Put/seqid=0 2024-11-10T03:17:55,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742237_1413 (size=12301) 2024-11-10T03:17:55,344 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/B/22ad3ae2088745658a17fc0386b650ec 2024-11-10T03:17:55,363 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/C/c503f34ca2844c40bd21884a3c31a5af is 50, key is test_row_0/C:col10/1731208674880/Put/seqid=0 2024-11-10T03:17:55,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742238_1414 (size=12301) 2024-11-10T03:17:55,371 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/C/c503f34ca2844c40bd21884a3c31a5af 2024-11-10T03:17:55,374 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/A/9afd9eff4e5d4caf8726330b3c2af5db as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/9afd9eff4e5d4caf8726330b3c2af5db 2024-11-10T03:17:55,380 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/9afd9eff4e5d4caf8726330b3c2af5db, entries=200, sequenceid=273, filesize=39.0 K 2024-11-10T03:17:55,381 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/B/22ad3ae2088745658a17fc0386b650ec as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/22ad3ae2088745658a17fc0386b650ec 2024-11-10T03:17:55,388 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/22ad3ae2088745658a17fc0386b650ec, entries=150, sequenceid=273, filesize=12.0 K 2024-11-10T03:17:55,389 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/C/c503f34ca2844c40bd21884a3c31a5af as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/c503f34ca2844c40bd21884a3c31a5af 2024-11-10T03:17:55,393 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/c503f34ca2844c40bd21884a3c31a5af, entries=150, sequenceid=273, filesize=12.0 K 2024-11-10T03:17:55,394 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for c6879c5307a3686e6731f46d96d3ca08 in 510ms, sequenceid=273, compaction requested=true 2024-11-10T03:17:55,394 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c6879c5307a3686e6731f46d96d3ca08: 2024-11-10T03:17:55,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c6879c5307a3686e6731f46d96d3ca08:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T03:17:55,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:17:55,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c6879c5307a3686e6731f46d96d3ca08:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T03:17:55,394 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T03:17:55,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:17:55,394 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T03:17:55,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c6879c5307a3686e6731f46d96d3ca08:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T03:17:55,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T03:17:55,395 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102828 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T03:17:55,395 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1541): c6879c5307a3686e6731f46d96d3ca08/A is initiating minor compaction (all files) 2024-11-10T03:17:55,395 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c6879c5307a3686e6731f46d96d3ca08/A in TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:55,395 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/1129c49e7cd4436684186c32375c56d0, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/87d156ff19d44e99b2ee2df8e93ccaa3, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/9afd9eff4e5d4caf8726330b3c2af5db] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp, totalSize=100.4 K 2024-11-10T03:17:55,395 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46888 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T03:17:55,395 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:55,395 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1541): c6879c5307a3686e6731f46d96d3ca08/B is initiating minor compaction (all files) 2024-11-10T03:17:55,395 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. files: [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/1129c49e7cd4436684186c32375c56d0, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/87d156ff19d44e99b2ee2df8e93ccaa3, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/9afd9eff4e5d4caf8726330b3c2af5db] 2024-11-10T03:17:55,395 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c6879c5307a3686e6731f46d96d3ca08/B in TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:55,395 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/8e38bf8df49a4e3da25345fb2993fa16, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/cf91029945ef4b14bf41dc548b555934, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/cfccbadf71214c25ba968465be75bffe, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/22ad3ae2088745658a17fc0386b650ec] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp, totalSize=45.8 K 2024-11-10T03:17:55,396 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1129c49e7cd4436684186c32375c56d0, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731208670544 2024-11-10T03:17:55,396 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e38bf8df49a4e3da25345fb2993fa16, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1731208670544 2024-11-10T03:17:55,396 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 87d156ff19d44e99b2ee2df8e93ccaa3, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1731208671450 2024-11-10T03:17:55,396 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting cf91029945ef4b14bf41dc548b555934, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731208670655 2024-11-10T03:17:55,397 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9afd9eff4e5d4caf8726330b3c2af5db, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731208672668 2024-11-10T03:17:55,397 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting cfccbadf71214c25ba968465be75bffe, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1731208671450 2024-11-10T03:17:55,397 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 22ad3ae2088745658a17fc0386b650ec, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731208672668 2024-11-10T03:17:55,419 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c6879c5307a3686e6731f46d96d3ca08] 2024-11-10T03:17:55,421 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c6879c5307a3686e6731f46d96d3ca08#B#compaction#345 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:17:55,421 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/B/45db82e440dc46ffa3aeaae0cd957e71 is 50, key is test_row_0/B:col10/1731208674880/Put/seqid=0 2024-11-10T03:17:55,422 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111036233f2af902455aa75a71254415a831_c6879c5307a3686e6731f46d96d3ca08 store=[table=TestAcidGuarantees family=A region=c6879c5307a3686e6731f46d96d3ca08] 2024-11-10T03:17:55,425 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111036233f2af902455aa75a71254415a831_c6879c5307a3686e6731f46d96d3ca08, store=[table=TestAcidGuarantees family=A region=c6879c5307a3686e6731f46d96d3ca08] 2024-11-10T03:17:55,425 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111036233f2af902455aa75a71254415a831_c6879c5307a3686e6731f46d96d3ca08 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c6879c5307a3686e6731f46d96d3ca08] 2024-11-10T03:17:55,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742239_1415 (size=12915) 2024-11-10T03:17:55,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742240_1416 (size=4469) 2024-11-10T03:17:55,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on c6879c5307a3686e6731f46d96d3ca08 2024-11-10T03:17:55,621 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c6879c5307a3686e6731f46d96d3ca08 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-10T03:17:55,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6879c5307a3686e6731f46d96d3ca08, store=A 2024-11-10T03:17:55,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:17:55,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6879c5307a3686e6731f46d96d3ca08, store=B 2024-11-10T03:17:55,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:17:55,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6879c5307a3686e6731f46d96d3ca08, store=C 2024-11-10T03:17:55,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:17:55,629 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110cb7c37c9280a4b048deb28eec40ca1cd_c6879c5307a3686e6731f46d96d3ca08 is 50, key is test_row_0/A:col10/1731208675619/Put/seqid=0 2024-11-10T03:17:55,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742241_1417 (size=14994) 2024-11-10T03:17:55,669 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:55,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53958 deadline: 1731208735658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:55,670 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:55,670 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:55,670 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:55,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:55,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53984 deadline: 1731208735667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:55,672 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:55,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53924 deadline: 1731208735668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:55,672 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:55,672 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:55,672 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:55,672 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:55,672 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:55,672 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:55,731 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:55,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53998 deadline: 1731208735723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:55,732 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 394d2222f724,38139,1731208572054 for 'test_row_2' in c6879c5307a3686e6731f46d96d3ca08 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4246 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T03:17:55,732 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:55,732 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:55,732 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:55,779 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:55,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53958 deadline: 1731208735773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:55,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:55,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:55,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:55,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:55,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53984 deadline: 1731208735783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:55,789 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:55,789 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:55,789 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:55,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53924 deadline: 1731208735783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:55,789 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:55,789 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:55,789 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:55,789 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:55,844 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c6879c5307a3686e6731f46d96d3ca08#A#compaction#346 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:17:55,844 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/A/c9f4223f5fc04c84bb5bd0212162038c is 175, key is test_row_0/A:col10/1731208674880/Put/seqid=0 2024-11-10T03:17:55,846 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/B/45db82e440dc46ffa3aeaae0cd957e71 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/45db82e440dc46ffa3aeaae0cd957e71 2024-11-10T03:17:55,855 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c6879c5307a3686e6731f46d96d3ca08/B of c6879c5307a3686e6731f46d96d3ca08 into 45db82e440dc46ffa3aeaae0cd957e71(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:17:55,855 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c6879c5307a3686e6731f46d96d3ca08: 2024-11-10T03:17:55,855 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., storeName=c6879c5307a3686e6731f46d96d3ca08/B, priority=12, startTime=1731208675394; duration=0sec 2024-11-10T03:17:55,855 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T03:17:55,855 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c6879c5307a3686e6731f46d96d3ca08:B 2024-11-10T03:17:55,855 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T03:17:55,857 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46888 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T03:17:55,857 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1541): c6879c5307a3686e6731f46d96d3ca08/C is initiating minor compaction (all files) 2024-11-10T03:17:55,857 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c6879c5307a3686e6731f46d96d3ca08/C in TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:55,857 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/fdfeda849bc74df286979cf33924ef3b, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/ce67747827d54eb8bbc394449199d798, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/4a9152e5371d4d0fb00ec7bbf1a332a7, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/c503f34ca2844c40bd21884a3c31a5af] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp, totalSize=45.8 K 2024-11-10T03:17:55,857 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting fdfeda849bc74df286979cf33924ef3b, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1731208670544 2024-11-10T03:17:55,858 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting ce67747827d54eb8bbc394449199d798, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731208670655 2024-11-10T03:17:55,858 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a9152e5371d4d0fb00ec7bbf1a332a7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1731208671450 2024-11-10T03:17:55,858 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting c503f34ca2844c40bd21884a3c31a5af, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731208672668 2024-11-10T03:17:55,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742242_1418 (size=31869) 2024-11-10T03:17:55,876 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c6879c5307a3686e6731f46d96d3ca08#C#compaction#348 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:17:55,877 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/C/65b40767936e47d6901cdc1fa203155c is 50, key is test_row_0/C:col10/1731208674880/Put/seqid=0 2024-11-10T03:17:55,881 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/A/c9f4223f5fc04c84bb5bd0212162038c as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/c9f4223f5fc04c84bb5bd0212162038c 2024-11-10T03:17:55,888 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c6879c5307a3686e6731f46d96d3ca08/A of c6879c5307a3686e6731f46d96d3ca08 into c9f4223f5fc04c84bb5bd0212162038c(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:17:55,888 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c6879c5307a3686e6731f46d96d3ca08: 2024-11-10T03:17:55,888 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., storeName=c6879c5307a3686e6731f46d96d3ca08/A, priority=13, startTime=1731208675394; duration=0sec 2024-11-10T03:17:55,888 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:17:55,888 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c6879c5307a3686e6731f46d96d3ca08:A 2024-11-10T03:17:55,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742243_1419 (size=12915) 2024-11-10T03:17:55,897 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/C/65b40767936e47d6901cdc1fa203155c as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/65b40767936e47d6901cdc1fa203155c 2024-11-10T03:17:55,906 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c6879c5307a3686e6731f46d96d3ca08/C of c6879c5307a3686e6731f46d96d3ca08 into 65b40767936e47d6901cdc1fa203155c(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:17:55,906 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c6879c5307a3686e6731f46d96d3ca08: 2024-11-10T03:17:55,906 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., storeName=c6879c5307a3686e6731f46d96d3ca08/C, priority=12, startTime=1731208675394; duration=0sec 2024-11-10T03:17:55,906 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:17:55,906 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c6879c5307a3686e6731f46d96d3ca08:C 2024-11-10T03:17:55,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-10T03:17:55,953 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T03:17:55,954 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T03:17:55,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-10T03:17:55,956 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T03:17:55,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-10T03:17:55,956 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T03:17:55,957 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T03:17:55,987 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:55,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53958 deadline: 1731208735984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:55,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:55,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:55,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:55,999 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:55,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53984 deadline: 1731208735993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:55,999 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:55,999 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:55,999 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:55,999 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:56,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53924 deadline: 1731208735993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:56,000 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:56,000 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:56,000 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:56,032 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:56,034 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110cb7c37c9280a4b048deb28eec40ca1cd_c6879c5307a3686e6731f46d96d3ca08 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110cb7c37c9280a4b048deb28eec40ca1cd_c6879c5307a3686e6731f46d96d3ca08 2024-11-10T03:17:56,035 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/A/27338101b7924542a003a29221afc034, store: [table=TestAcidGuarantees family=A region=c6879c5307a3686e6731f46d96d3ca08] 2024-11-10T03:17:56,036 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/A/27338101b7924542a003a29221afc034 is 175, key is test_row_0/A:col10/1731208675619/Put/seqid=0 2024-11-10T03:17:56,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742244_1420 (size=39949) 2024-11-10T03:17:56,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-10T03:17:56,109 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-10T03:17:56,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:56,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. as already flushing 2024-11-10T03:17:56,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:56,110 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:56,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:56,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:56,262 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-10T03:17:56,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:56,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. as already flushing 2024-11-10T03:17:56,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:56,263 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:56,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:56,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:56,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-10T03:17:56,297 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:56,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53958 deadline: 1731208736293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:56,298 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:56,298 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:56,298 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:56,306 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:56,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53984 deadline: 1731208736302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:56,306 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:56,306 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:56,307 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:56,307 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:56,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53924 deadline: 1731208736302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:56,307 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:56,307 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:56,307 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:56,415 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-10T03:17:56,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:56,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. as already flushing 2024-11-10T03:17:56,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:56,415 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:56,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:56,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:56,440 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=293, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/A/27338101b7924542a003a29221afc034 2024-11-10T03:17:56,448 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/B/92eea1336ddf4297b9796d2693a87c46 is 50, key is test_row_0/B:col10/1731208675619/Put/seqid=0 2024-11-10T03:17:56,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742245_1421 (size=12301) 2024-11-10T03:17:56,568 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-10T03:17:56,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:56,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. as already flushing 2024-11-10T03:17:56,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:56,568 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:56,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:56,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:56,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-10T03:17:56,721 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-10T03:17:56,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:56,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. as already flushing 2024-11-10T03:17:56,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:56,721 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:56,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:56,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:56,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:56,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53958 deadline: 1731208736804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:56,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:56,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:56,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:56,816 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:56,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53984 deadline: 1731208736813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:56,817 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:56,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:56,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:56,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:56,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53924 deadline: 1731208736813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:56,817 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:56,817 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:56,817 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:56,855 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/B/92eea1336ddf4297b9796d2693a87c46 2024-11-10T03:17:56,867 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/C/40a8e01cbe724da798e74c1d8a0fd9e0 is 50, key is test_row_0/C:col10/1731208675619/Put/seqid=0 2024-11-10T03:17:56,874 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-10T03:17:56,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:56,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. as already flushing 2024-11-10T03:17:56,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:56,875 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:56,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:56,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:56,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742246_1422 (size=12301) 2024-11-10T03:17:57,030 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-10T03:17:57,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:57,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. as already flushing 2024-11-10T03:17:57,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:57,031 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:57,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:57,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:57,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-10T03:17:57,192 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-10T03:17:57,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:57,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. as already flushing 2024-11-10T03:17:57,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:57,193 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:57,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:57,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:57,288 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/C/40a8e01cbe724da798e74c1d8a0fd9e0 2024-11-10T03:17:57,297 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/A/27338101b7924542a003a29221afc034 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/27338101b7924542a003a29221afc034 2024-11-10T03:17:57,302 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/27338101b7924542a003a29221afc034, entries=200, sequenceid=293, filesize=39.0 K 2024-11-10T03:17:57,303 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/B/92eea1336ddf4297b9796d2693a87c46 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/92eea1336ddf4297b9796d2693a87c46 2024-11-10T03:17:57,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,309 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/92eea1336ddf4297b9796d2693a87c46, entries=150, sequenceid=293, filesize=12.0 K 2024-11-10T03:17:57,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,310 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/C/40a8e01cbe724da798e74c1d8a0fd9e0 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/40a8e01cbe724da798e74c1d8a0fd9e0 2024-11-10T03:17:57,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,314 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/40a8e01cbe724da798e74c1d8a0fd9e0, entries=150, sequenceid=293, filesize=12.0 K 2024-11-10T03:17:57,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,315 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for c6879c5307a3686e6731f46d96d3ca08 in 1694ms, sequenceid=293, compaction requested=false 2024-11-10T03:17:57,315 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c6879c5307a3686e6731f46d96d3ca08: 2024-11-10T03:17:57,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,345 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-10T03:17:57,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:57,345 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing c6879c5307a3686e6731f46d96d3ca08 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-10T03:17:57,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6879c5307a3686e6731f46d96d3ca08, store=A 2024-11-10T03:17:57,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:17:57,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6879c5307a3686e6731f46d96d3ca08, store=B 2024-11-10T03:17:57,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:17:57,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6879c5307a3686e6731f46d96d3ca08, store=C 2024-11-10T03:17:57,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:17:57,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110775f0a23d1d045e4832e8bacb37a9289_c6879c5307a3686e6731f46d96d3ca08 is 50, key is test_row_1/A:col10/1731208675665/Put/seqid=0 2024-11-10T03:17:57,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742247_1423 (size=9914) 2024-11-10T03:17:57,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,403 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110775f0a23d1d045e4832e8bacb37a9289_c6879c5307a3686e6731f46d96d3ca08 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110775f0a23d1d045e4832e8bacb37a9289_c6879c5307a3686e6731f46d96d3ca08 2024-11-10T03:17:57,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/A/05b07ee73b7541b686d8b1aafe726c0e, store: [table=TestAcidGuarantees family=A region=c6879c5307a3686e6731f46d96d3ca08] 2024-11-10T03:17:57,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/A/05b07ee73b7541b686d8b1aafe726c0e is 175, key is test_row_1/A:col10/1731208675665/Put/seqid=0 2024-11-10T03:17:57,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742248_1424 (size=22561) 2024-11-10T03:17:57,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,413 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=308, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/A/05b07ee73b7541b686d8b1aafe726c0e 2024-11-10T03:17:57,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/B/8ca50efbc7a5419eb0705f36601f9252 is 50, key is test_row_1/B:col10/1731208675665/Put/seqid=0 2024-11-10T03:17:57,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742249_1425 (size=9857) 2024-11-10T03:17:57,430 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/B/8ca50efbc7a5419eb0705f36601f9252 2024-11-10T03:17:57,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/C/31939a83d581466fb9710a7fcf69daf5 is 50, key is test_row_1/C:col10/1731208675665/Put/seqid=0 2024-11-10T03:17:57,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742250_1426 (size=9857) 2024-11-10T03:17:57,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on c6879c5307a3686e6731f46d96d3ca08 2024-11-10T03:17:57,831 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. as already flushing 2024-11-10T03:17:57,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,845 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/C/31939a83d581466fb9710a7fcf69daf5 2024-11-10T03:17:57,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/A/05b07ee73b7541b686d8b1aafe726c0e as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/05b07ee73b7541b686d8b1aafe726c0e 2024-11-10T03:17:57,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,853 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/05b07ee73b7541b686d8b1aafe726c0e, entries=100, sequenceid=308, filesize=22.0 K 2024-11-10T03:17:57,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/B/8ca50efbc7a5419eb0705f36601f9252 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/8ca50efbc7a5419eb0705f36601f9252 2024-11-10T03:17:57,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,865 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/8ca50efbc7a5419eb0705f36601f9252, entries=100, sequenceid=308, filesize=9.6 K 2024-11-10T03:17:57,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/C/31939a83d581466fb9710a7fcf69daf5 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/31939a83d581466fb9710a7fcf69daf5 2024-11-10T03:17:57,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,870 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/31939a83d581466fb9710a7fcf69daf5, entries=100, sequenceid=308, filesize=9.6 K 2024-11-10T03:17:57,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,871 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=6.71 KB/6870 for c6879c5307a3686e6731f46d96d3ca08 in 526ms, sequenceid=308, compaction requested=true 2024-11-10T03:17:57,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for c6879c5307a3686e6731f46d96d3ca08: 2024-11-10T03:17:57,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:57,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-10T03:17:57,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-10T03:17:57,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,874 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-10T03:17:57,874 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9150 sec 2024-11-10T03:17:57,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,875 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 1.9200 sec 2024-11-10T03:17:57,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on c6879c5307a3686e6731f46d96d3ca08 2024-11-10T03:17:57,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,935 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c6879c5307a3686e6731f46d96d3ca08 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-10T03:17:57,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6879c5307a3686e6731f46d96d3ca08, store=A 2024-11-10T03:17:57,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:17:57,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6879c5307a3686e6731f46d96d3ca08, store=B 2024-11-10T03:17:57,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:17:57,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6879c5307a3686e6731f46d96d3ca08, store=C 2024-11-10T03:17:57,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:17:57,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,951 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110ecf133942456486993a2c3619f0d32cb_c6879c5307a3686e6731f46d96d3ca08 is 50, key is test_row_0/A:col10/1731208677930/Put/seqid=0 2024-11-10T03:17:57,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742251_1427 (size=17534) 2024-11-10T03:17:57,979 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:57,985 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110ecf133942456486993a2c3619f0d32cb_c6879c5307a3686e6731f46d96d3ca08 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110ecf133942456486993a2c3619f0d32cb_c6879c5307a3686e6731f46d96d3ca08 2024-11-10T03:17:57,986 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/A/473fca4c8c6f48f5ba97dcf6289fe18b, store: [table=TestAcidGuarantees family=A region=c6879c5307a3686e6731f46d96d3ca08] 2024-11-10T03:17:57,988 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/A/473fca4c8c6f48f5ba97dcf6289fe18b is 175, key is test_row_0/A:col10/1731208677930/Put/seqid=0 2024-11-10T03:17:58,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742252_1428 (size=48639) 2024-11-10T03:17:58,047 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731208678026, max timestamp=1731208678044Num uniques 859; 2024-11-10T03:17:58,047 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731208678026, max timestamp=1731208678044Num uniques 859; 2024-11-10T03:17:58,047 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T03:17:58,047 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T03:17:58,049 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731208678026, max timestamp=1731208678044Num uniques 859; 2024-11-10T03:17:58,049 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T03:17:58,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-10T03:17:58,103 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T03:17:58,103 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T03:17:58,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-10T03:17:58,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-10T03:17:58,105 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T03:17:58,105 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T03:17:58,106 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T03:17:58,127 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:58,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53984 deadline: 1731208738119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:58,127 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:58,127 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:58,127 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:58,139 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:58,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53924 deadline: 1731208738121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:58,139 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:58,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53958 deadline: 1731208738122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:58,139 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:58,139 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:58,139 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:58,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:58,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:58,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:58,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-10T03:17:58,240 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:58,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53984 deadline: 1731208738233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:58,240 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:58,240 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:58,240 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:58,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:58,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53924 deadline: 1731208738243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:58,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:58,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53958 deadline: 1731208738243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:58,259 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:58,259 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:58,259 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:58,262 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:58,262 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:58,262 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:58,263 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-10T03:17:58,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:58,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. as already flushing 2024-11-10T03:17:58,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:58,263 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:58,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:58,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:58,410 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=318, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/A/473fca4c8c6f48f5ba97dcf6289fe18b 2024-11-10T03:17:58,417 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/B/03ceb841cc814322b2ce6bb1482fdab1 is 50, key is test_row_0/B:col10/1731208677930/Put/seqid=0 2024-11-10T03:17:58,421 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-10T03:17:58,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:58,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. as already flushing 2024-11-10T03:17:58,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:58,421 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:58,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:58,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:58,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-10T03:17:58,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742253_1429 (size=12301) 2024-11-10T03:17:58,458 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:58,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53984 deadline: 1731208738443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:58,458 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:58,458 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:58,458 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:58,478 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:58,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53958 deadline: 1731208738462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:58,479 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:58,479 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:58,479 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:58,479 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:58,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53924 deadline: 1731208738473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:58,479 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:58,479 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:58,479 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:58,574 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-10T03:17:58,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:58,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. as already flushing 2024-11-10T03:17:58,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:58,574 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:58,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:58,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:58,727 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-10T03:17:58,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:58,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. as already flushing 2024-11-10T03:17:58,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:58,727 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:58,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:58,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:58,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-10T03:17:58,778 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:58,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53984 deadline: 1731208738763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:58,778 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:58,778 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:58,778 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:58,805 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:58,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53958 deadline: 1731208738783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:58,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:58,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:58,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:58,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:58,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53924 deadline: 1731208738792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:58,806 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:58,807 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:58,807 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:58,834 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/B/03ceb841cc814322b2ce6bb1482fdab1 2024-11-10T03:17:58,860 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/C/b1371b51bcc245c8a4d8768bafbe5a1b is 50, key is test_row_0/C:col10/1731208677930/Put/seqid=0 2024-11-10T03:17:58,881 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-10T03:17:58,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:58,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. as already flushing 2024-11-10T03:17:58,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:58,881 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:58,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:58,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:17:58,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742254_1430 (size=12301) 2024-11-10T03:17:58,892 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/C/b1371b51bcc245c8a4d8768bafbe5a1b 2024-11-10T03:17:58,900 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/A/473fca4c8c6f48f5ba97dcf6289fe18b as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/473fca4c8c6f48f5ba97dcf6289fe18b 2024-11-10T03:17:58,904 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/473fca4c8c6f48f5ba97dcf6289fe18b, entries=250, sequenceid=318, filesize=47.5 K 2024-11-10T03:17:58,905 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/B/03ceb841cc814322b2ce6bb1482fdab1 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/03ceb841cc814322b2ce6bb1482fdab1 2024-11-10T03:17:58,922 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/03ceb841cc814322b2ce6bb1482fdab1, entries=150, sequenceid=318, filesize=12.0 K 2024-11-10T03:17:58,923 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/C/b1371b51bcc245c8a4d8768bafbe5a1b as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/b1371b51bcc245c8a4d8768bafbe5a1b 2024-11-10T03:17:58,935 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/b1371b51bcc245c8a4d8768bafbe5a1b, entries=150, sequenceid=318, filesize=12.0 K 2024-11-10T03:17:58,936 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for c6879c5307a3686e6731f46d96d3ca08 in 1001ms, sequenceid=318, compaction requested=true 2024-11-10T03:17:58,936 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c6879c5307a3686e6731f46d96d3ca08: 2024-11-10T03:17:58,936 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T03:17:58,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c6879c5307a3686e6731f46d96d3ca08:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T03:17:58,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:17:58,937 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T03:17:58,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c6879c5307a3686e6731f46d96d3ca08:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T03:17:58,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:17:58,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c6879c5307a3686e6731f46d96d3ca08:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T03:17:58,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T03:17:58,939 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 143018 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T03:17:58,939 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1541): c6879c5307a3686e6731f46d96d3ca08/A is initiating minor compaction (all files) 2024-11-10T03:17:58,939 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c6879c5307a3686e6731f46d96d3ca08/A in TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:58,939 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/c9f4223f5fc04c84bb5bd0212162038c, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/27338101b7924542a003a29221afc034, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/05b07ee73b7541b686d8b1aafe726c0e, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/473fca4c8c6f48f5ba97dcf6289fe18b] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp, totalSize=139.7 K 2024-11-10T03:17:58,939 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:58,939 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. files: [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/c9f4223f5fc04c84bb5bd0212162038c, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/27338101b7924542a003a29221afc034, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/05b07ee73b7541b686d8b1aafe726c0e, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/473fca4c8c6f48f5ba97dcf6289fe18b] 2024-11-10T03:17:58,939 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47374 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T03:17:58,939 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1541): c6879c5307a3686e6731f46d96d3ca08/B is initiating minor compaction (all files) 2024-11-10T03:17:58,939 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c6879c5307a3686e6731f46d96d3ca08/B in TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:58,939 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/45db82e440dc46ffa3aeaae0cd957e71, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/92eea1336ddf4297b9796d2693a87c46, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/8ca50efbc7a5419eb0705f36601f9252, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/03ceb841cc814322b2ce6bb1482fdab1] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp, totalSize=46.3 K 2024-11-10T03:17:58,940 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 45db82e440dc46ffa3aeaae0cd957e71, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731208672668 2024-11-10T03:17:58,940 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 92eea1336ddf4297b9796d2693a87c46, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731208674957 2024-11-10T03:17:58,940 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting c9f4223f5fc04c84bb5bd0212162038c, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731208672668 2024-11-10T03:17:58,940 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ca50efbc7a5419eb0705f36601f9252, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1731208675665 2024-11-10T03:17:58,941 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 27338101b7924542a003a29221afc034, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731208674956 2024-11-10T03:17:58,941 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 03ceb841cc814322b2ce6bb1482fdab1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1731208677908 2024-11-10T03:17:58,941 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 05b07ee73b7541b686d8b1aafe726c0e, keycount=100, bloomtype=ROW, size=22.0 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1731208675665 2024-11-10T03:17:58,942 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 473fca4c8c6f48f5ba97dcf6289fe18b, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1731208677907 2024-11-10T03:17:58,957 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c6879c5307a3686e6731f46d96d3ca08] 2024-11-10T03:17:58,959 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c6879c5307a3686e6731f46d96d3ca08#B#compaction#357 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:17:58,959 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/B/0f1efa95b009435882d6333a91a5c135 is 50, key is test_row_0/B:col10/1731208677930/Put/seqid=0 2024-11-10T03:17:58,959 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241110b1556419079b4497ae56c46b626e34e4_c6879c5307a3686e6731f46d96d3ca08 store=[table=TestAcidGuarantees family=A region=c6879c5307a3686e6731f46d96d3ca08] 2024-11-10T03:17:58,962 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241110b1556419079b4497ae56c46b626e34e4_c6879c5307a3686e6731f46d96d3ca08, store=[table=TestAcidGuarantees family=A region=c6879c5307a3686e6731f46d96d3ca08] 2024-11-10T03:17:58,962 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110b1556419079b4497ae56c46b626e34e4_c6879c5307a3686e6731f46d96d3ca08 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c6879c5307a3686e6731f46d96d3ca08] 2024-11-10T03:17:59,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742256_1432 (size=4469) 2024-11-10T03:17:59,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742255_1431 (size=13051) 2024-11-10T03:17:59,035 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-10T03:17:59,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:59,035 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing c6879c5307a3686e6731f46d96d3ca08 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-10T03:17:59,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6879c5307a3686e6731f46d96d3ca08, store=A 2024-11-10T03:17:59,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T03:17:59,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6879c5307a3686e6731f46d96d3ca08, store=B 2024-11-10T03:17:59,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T03:17:59,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6879c5307a3686e6731f46d96d3ca08, store=C 2024-11-10T03:17:59,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T03:17:59,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111037d5aba0f003494d87a7d903c81428d7_c6879c5307a3686e6731f46d96d3ca08 is 50, key is test_row_0/A:col10/1731208678076/Put/seqid=0 2024-11-10T03:17:59,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742257_1433 (size=12454) 2024-11-10T03:17:59,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-10T03:17:59,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on c6879c5307a3686e6731f46d96d3ca08 2024-11-10T03:17:59,293 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. as already flushing 2024-11-10T03:17:59,332 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:59,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53958 deadline: 1731208739323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:59,332 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:59,332 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:59,332 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:59,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53924 deadline: 1731208739324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:59,332 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:59,332 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:59,332 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:59,332 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:59,338 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:59,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53984 deadline: 1731208739325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:59,338 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:59,338 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:59,338 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:59,424 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c6879c5307a3686e6731f46d96d3ca08#A#compaction#358 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:17:59,425 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/A/38b601ce04ae4ac89c3acba38095611a is 175, key is test_row_0/A:col10/1731208677930/Put/seqid=0 2024-11-10T03:17:59,429 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/B/0f1efa95b009435882d6333a91a5c135 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/0f1efa95b009435882d6333a91a5c135 2024-11-10T03:17:59,433 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c6879c5307a3686e6731f46d96d3ca08/B of c6879c5307a3686e6731f46d96d3ca08 into 0f1efa95b009435882d6333a91a5c135(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:17:59,433 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c6879c5307a3686e6731f46d96d3ca08: 2024-11-10T03:17:59,433 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., storeName=c6879c5307a3686e6731f46d96d3ca08/B, priority=12, startTime=1731208678937; duration=0sec 2024-11-10T03:17:59,433 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T03:17:59,433 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c6879c5307a3686e6731f46d96d3ca08:B 2024-11-10T03:17:59,434 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T03:17:59,435 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47374 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T03:17:59,435 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1541): c6879c5307a3686e6731f46d96d3ca08/C is initiating minor compaction (all files) 2024-11-10T03:17:59,435 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c6879c5307a3686e6731f46d96d3ca08/C in TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:59,435 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/65b40767936e47d6901cdc1fa203155c, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/40a8e01cbe724da798e74c1d8a0fd9e0, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/31939a83d581466fb9710a7fcf69daf5, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/b1371b51bcc245c8a4d8768bafbe5a1b] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp, totalSize=46.3 K 2024-11-10T03:17:59,435 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 65b40767936e47d6901cdc1fa203155c, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731208672668 2024-11-10T03:17:59,435 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 40a8e01cbe724da798e74c1d8a0fd9e0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731208674957 2024-11-10T03:17:59,436 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 31939a83d581466fb9710a7fcf69daf5, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1731208675665 2024-11-10T03:17:59,436 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting b1371b51bcc245c8a4d8768bafbe5a1b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1731208677908 2024-11-10T03:17:59,456 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:59,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53984 deadline: 1731208739443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:59,457 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:59,457 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:59,457 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:59,457 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:59,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53958 deadline: 1731208739448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:59,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:59,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:59,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:59,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:17:59,458 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:59,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53924 deadline: 1731208739448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:59,461 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:59,461 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:59,461 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:59,462 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111037d5aba0f003494d87a7d903c81428d7_c6879c5307a3686e6731f46d96d3ca08 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111037d5aba0f003494d87a7d903c81428d7_c6879c5307a3686e6731f46d96d3ca08 2024-11-10T03:17:59,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/A/ece281eddcd5458890469941757c5ad6, store: [table=TestAcidGuarantees family=A region=c6879c5307a3686e6731f46d96d3ca08] 2024-11-10T03:17:59,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/A/ece281eddcd5458890469941757c5ad6 is 175, key is test_row_0/A:col10/1731208678076/Put/seqid=0 2024-11-10T03:17:59,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742258_1434 (size=32005) 2024-11-10T03:17:59,476 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/A/38b601ce04ae4ac89c3acba38095611a as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/38b601ce04ae4ac89c3acba38095611a 2024-11-10T03:17:59,481 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c6879c5307a3686e6731f46d96d3ca08#C#compaction#360 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:17:59,482 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/C/06fa856c30f043b7b5d7bb26bc88b40c is 50, key is test_row_0/C:col10/1731208677930/Put/seqid=0 2024-11-10T03:17:59,483 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c6879c5307a3686e6731f46d96d3ca08/A of c6879c5307a3686e6731f46d96d3ca08 into 38b601ce04ae4ac89c3acba38095611a(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:17:59,483 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c6879c5307a3686e6731f46d96d3ca08: 2024-11-10T03:17:59,483 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., storeName=c6879c5307a3686e6731f46d96d3ca08/A, priority=12, startTime=1731208678936; duration=0sec 2024-11-10T03:17:59,483 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:17:59,483 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c6879c5307a3686e6731f46d96d3ca08:A 2024-11-10T03:17:59,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742259_1435 (size=31255) 2024-11-10T03:17:59,504 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=354, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/A/ece281eddcd5458890469941757c5ad6 2024-11-10T03:17:59,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/B/c8a4ff9d7aba4f179d8cfa61b4aa30af is 50, key is test_row_0/B:col10/1731208678076/Put/seqid=0 2024-11-10T03:17:59,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742260_1436 (size=13051) 2024-11-10T03:17:59,540 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/C/06fa856c30f043b7b5d7bb26bc88b40c as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/06fa856c30f043b7b5d7bb26bc88b40c 2024-11-10T03:17:59,548 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c6879c5307a3686e6731f46d96d3ca08/C of c6879c5307a3686e6731f46d96d3ca08 into 06fa856c30f043b7b5d7bb26bc88b40c(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:17:59,548 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c6879c5307a3686e6731f46d96d3ca08: 2024-11-10T03:17:59,548 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., storeName=c6879c5307a3686e6731f46d96d3ca08/C, priority=12, startTime=1731208678937; duration=0sec 2024-11-10T03:17:59,548 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:17:59,548 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c6879c5307a3686e6731f46d96d3ca08:C 2024-11-10T03:17:59,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742261_1437 (size=12301) 2024-11-10T03:17:59,561 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/B/c8a4ff9d7aba4f179d8cfa61b4aa30af 2024-11-10T03:17:59,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/C/df6a368c4f5c4a68b57e43513ff8fdca is 50, key is test_row_0/C:col10/1731208678076/Put/seqid=0 2024-11-10T03:17:59,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742262_1438 (size=12301) 2024-11-10T03:17:59,630 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/C/df6a368c4f5c4a68b57e43513ff8fdca 2024-11-10T03:17:59,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/A/ece281eddcd5458890469941757c5ad6 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/ece281eddcd5458890469941757c5ad6 2024-11-10T03:17:59,641 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/ece281eddcd5458890469941757c5ad6, entries=150, sequenceid=354, filesize=30.5 K 2024-11-10T03:17:59,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/B/c8a4ff9d7aba4f179d8cfa61b4aa30af as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/c8a4ff9d7aba4f179d8cfa61b4aa30af 2024-11-10T03:17:59,651 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/c8a4ff9d7aba4f179d8cfa61b4aa30af, entries=150, sequenceid=354, filesize=12.0 K 2024-11-10T03:17:59,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/C/df6a368c4f5c4a68b57e43513ff8fdca as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/df6a368c4f5c4a68b57e43513ff8fdca 2024-11-10T03:17:59,657 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/df6a368c4f5c4a68b57e43513ff8fdca, entries=150, sequenceid=354, filesize=12.0 K 2024-11-10T03:17:59,660 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for c6879c5307a3686e6731f46d96d3ca08 in 625ms, sequenceid=354, compaction requested=false 2024-11-10T03:17:59,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for c6879c5307a3686e6731f46d96d3ca08: 2024-11-10T03:17:59,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:17:59,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-10T03:17:59,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-10T03:17:59,670 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-10T03:17:59,671 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5620 sec 2024-11-10T03:17:59,677 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c6879c5307a3686e6731f46d96d3ca08 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-10T03:17:59,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6879c5307a3686e6731f46d96d3ca08, store=A 2024-11-10T03:17:59,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:17:59,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6879c5307a3686e6731f46d96d3ca08, store=B 2024-11-10T03:17:59,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:17:59,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6879c5307a3686e6731f46d96d3ca08, store=C 2024-11-10T03:17:59,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:17:59,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on c6879c5307a3686e6731f46d96d3ca08 2024-11-10T03:17:59,691 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 1.5720 sec 2024-11-10T03:17:59,702 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411109613421a4ffd47dcaa4a8aa274892325_c6879c5307a3686e6731f46d96d3ca08 is 50, key is test_row_0/A:col10/1731208679673/Put/seqid=0 2024-11-10T03:17:59,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742263_1439 (size=17534) 2024-11-10T03:17:59,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:59,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53998 deadline: 1731208739763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:59,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:59,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53958 deadline: 1731208739764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:59,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:59,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:59,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:59,781 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 394d2222f724,38139,1731208572054 for 'test_row_2' in c6879c5307a3686e6731f46d96d3ca08 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8295 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T03:17:59,782 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:59,782 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:59,782 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:59,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:59,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53924 deadline: 1731208739774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:59,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:59,789 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:59,789 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:59,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53984 deadline: 1731208739780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:59,789 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:59,789 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:59,789 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:59,789 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:59,884 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:59,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53958 deadline: 1731208739883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:59,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:59,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:59,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:59,898 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:59,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53924 deadline: 1731208739893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:59,899 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:59,899 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:59,899 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:17:59,899 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:17:59,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53984 deadline: 1731208739893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:17:59,899 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:59,899 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:17:59,899 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:00,099 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:00,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53958 deadline: 1731208740093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:00,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:00,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:00,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:00,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:00,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53924 deadline: 1731208740102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:00,112 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:00,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53984 deadline: 1731208740103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:00,113 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:00,113 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:00,113 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:00,114 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:00,114 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:00,114 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:00,142 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:18:00,146 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411109613421a4ffd47dcaa4a8aa274892325_c6879c5307a3686e6731f46d96d3ca08 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411109613421a4ffd47dcaa4a8aa274892325_c6879c5307a3686e6731f46d96d3ca08 2024-11-10T03:18:00,147 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/A/66d46b69b9124f80abe859f9e1ce41d1, store: [table=TestAcidGuarantees family=A region=c6879c5307a3686e6731f46d96d3ca08] 2024-11-10T03:18:00,148 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/A/66d46b69b9124f80abe859f9e1ce41d1 is 175, key is test_row_0/A:col10/1731208679673/Put/seqid=0 2024-11-10T03:18:00,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742264_1440 (size=48639) 2024-11-10T03:18:00,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-10T03:18:00,253 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T03:18:00,254 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T03:18:00,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-10T03:18:00,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-10T03:18:00,262 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T03:18:00,262 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T03:18:00,263 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T03:18:00,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-10T03:18:00,415 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-10T03:18:00,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:18:00,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. as already flushing 2024-11-10T03:18:00,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:18:00,415 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:00,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:00,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:00,416 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:00,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53958 deadline: 1731208740403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:00,417 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:00,418 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:00,418 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:00,433 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:00,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53984 deadline: 1731208740423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:00,433 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:00,433 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:00,433 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:00,433 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:00,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53924 deadline: 1731208740423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:00,434 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:00,434 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:00,434 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:00,555 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=369, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/A/66d46b69b9124f80abe859f9e1ce41d1 2024-11-10T03:18:00,568 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-10T03:18:00,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:18:00,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. as already flushing 2024-11-10T03:18:00,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:18:00,568 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:00,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:00,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:00,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-10T03:18:00,584 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/B/d357b34481bf4112bbf347c551c51798 is 50, key is test_row_0/B:col10/1731208679673/Put/seqid=0 2024-11-10T03:18:00,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742265_1441 (size=12301) 2024-11-10T03:18:00,724 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-10T03:18:00,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:18:00,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. as already flushing 2024-11-10T03:18:00,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:18:00,724 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:00,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:00,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:00,727 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37299 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=394d2222f724,38139,1731208572054, table=TestAcidGuarantees, region=c6879c5307a3686e6731f46d96d3ca08. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-10T03:18:00,881 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-10T03:18:00,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:18:00,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. as already flushing 2024-11-10T03:18:00,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:18:00,882 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:00,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:00,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:00,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-10T03:18:00,942 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:00,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53958 deadline: 1731208740932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:00,942 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:00,942 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:00,942 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:00,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:00,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53984 deadline: 1731208740943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:00,955 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:00,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53924 deadline: 1731208740944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:00,957 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:00,957 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:00,957 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:00,958 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:00,958 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:00,958 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:01,014 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/B/d357b34481bf4112bbf347c551c51798 2024-11-10T03:18:01,044 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/C/93651eb5b276480a89eb5d14e4377e3a is 50, key is test_row_0/C:col10/1731208679673/Put/seqid=0 2024-11-10T03:18:01,044 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-10T03:18:01,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:18:01,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. as already flushing 2024-11-10T03:18:01,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:18:01,044 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:01,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:01,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:01,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742266_1442 (size=12301) 2024-11-10T03:18:01,063 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/C/93651eb5b276480a89eb5d14e4377e3a 2024-11-10T03:18:01,073 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/A/66d46b69b9124f80abe859f9e1ce41d1 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/66d46b69b9124f80abe859f9e1ce41d1 2024-11-10T03:18:01,078 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/66d46b69b9124f80abe859f9e1ce41d1, entries=250, sequenceid=369, filesize=47.5 K 2024-11-10T03:18:01,078 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/B/d357b34481bf4112bbf347c551c51798 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/d357b34481bf4112bbf347c551c51798 2024-11-10T03:18:01,084 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/d357b34481bf4112bbf347c551c51798, entries=150, sequenceid=369, filesize=12.0 K 2024-11-10T03:18:01,087 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/C/93651eb5b276480a89eb5d14e4377e3a as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/93651eb5b276480a89eb5d14e4377e3a 2024-11-10T03:18:01,099 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/93651eb5b276480a89eb5d14e4377e3a, entries=150, sequenceid=369, filesize=12.0 K 2024-11-10T03:18:01,101 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for c6879c5307a3686e6731f46d96d3ca08 in 1425ms, sequenceid=369, compaction requested=true 2024-11-10T03:18:01,101 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c6879c5307a3686e6731f46d96d3ca08: 2024-11-10T03:18:01,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c6879c5307a3686e6731f46d96d3ca08:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T03:18:01,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T03:18:01,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c6879c5307a3686e6731f46d96d3ca08:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T03:18:01,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-10T03:18:01,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c6879c5307a3686e6731f46d96d3ca08:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T03:18:01,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-10T03:18:01,101 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T03:18:01,102 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T03:18:01,103 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T03:18:01,103 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111899 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T03:18:01,103 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1541): c6879c5307a3686e6731f46d96d3ca08/A is initiating minor compaction (all files) 2024-11-10T03:18:01,103 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1541): c6879c5307a3686e6731f46d96d3ca08/C is initiating minor compaction (all files) 2024-11-10T03:18:01,103 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c6879c5307a3686e6731f46d96d3ca08/C in TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:18:01,103 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c6879c5307a3686e6731f46d96d3ca08/A in TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:18:01,103 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/06fa856c30f043b7b5d7bb26bc88b40c, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/df6a368c4f5c4a68b57e43513ff8fdca, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/93651eb5b276480a89eb5d14e4377e3a] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp, totalSize=36.8 K 2024-11-10T03:18:01,103 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/38b601ce04ae4ac89c3acba38095611a, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/ece281eddcd5458890469941757c5ad6, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/66d46b69b9124f80abe859f9e1ce41d1] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp, totalSize=109.3 K 2024-11-10T03:18:01,103 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:18:01,103 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. files: [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/38b601ce04ae4ac89c3acba38095611a, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/ece281eddcd5458890469941757c5ad6, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/66d46b69b9124f80abe859f9e1ce41d1] 2024-11-10T03:18:01,111 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 38b601ce04ae4ac89c3acba38095611a, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1731208677908 2024-11-10T03:18:01,111 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 06fa856c30f043b7b5d7bb26bc88b40c, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1731208677908 2024-11-10T03:18:01,117 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting df6a368c4f5c4a68b57e43513ff8fdca, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1731208678076 2024-11-10T03:18:01,117 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting ece281eddcd5458890469941757c5ad6, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1731208678076 2024-11-10T03:18:01,118 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 66d46b69b9124f80abe859f9e1ce41d1, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1731208679305 2024-11-10T03:18:01,118 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 93651eb5b276480a89eb5d14e4377e3a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1731208679305 2024-11-10T03:18:01,130 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c6879c5307a3686e6731f46d96d3ca08#C#compaction#366 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:18:01,131 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/C/7db09b02ba284dba87b479b4a931011b is 50, key is test_row_0/C:col10/1731208679673/Put/seqid=0 2024-11-10T03:18:01,139 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c6879c5307a3686e6731f46d96d3ca08] 2024-11-10T03:18:01,158 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111001ebcaed63a940a6a72d0c738e0e08f2_c6879c5307a3686e6731f46d96d3ca08 store=[table=TestAcidGuarantees family=A region=c6879c5307a3686e6731f46d96d3ca08] 2024-11-10T03:18:01,160 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111001ebcaed63a940a6a72d0c738e0e08f2_c6879c5307a3686e6731f46d96d3ca08, store=[table=TestAcidGuarantees family=A region=c6879c5307a3686e6731f46d96d3ca08] 2024-11-10T03:18:01,161 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111001ebcaed63a940a6a72d0c738e0e08f2_c6879c5307a3686e6731f46d96d3ca08 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c6879c5307a3686e6731f46d96d3ca08] 2024-11-10T03:18:01,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742267_1443 (size=13153) 2024-11-10T03:18:01,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742268_1444 (size=4469) 2024-11-10T03:18:01,204 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c6879c5307a3686e6731f46d96d3ca08#A#compaction#367 average throughput is 0.38 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:18:01,205 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/A/f467615b58ad463782ddf3915cd5ec72 is 175, key is test_row_0/A:col10/1731208679673/Put/seqid=0 2024-11-10T03:18:01,208 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-10T03:18:01,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:18:01,209 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing c6879c5307a3686e6731f46d96d3ca08 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-10T03:18:01,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6879c5307a3686e6731f46d96d3ca08, store=A 2024-11-10T03:18:01,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:18:01,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6879c5307a3686e6731f46d96d3ca08, store=B 2024-11-10T03:18:01,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:18:01,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6879c5307a3686e6731f46d96d3ca08, store=C 2024-11-10T03:18:01,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:18:01,229 INFO [Thread-1572 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1572. 2024-11-10T03:18:01,229 DEBUG [Thread-1572 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T03:18:01,229 DEBUG [Thread-1572 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:18:01,229 DEBUG [Thread-1572 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:18:01,230 INFO [Thread-1570 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1570. 2024-11-10T03:18:01,230 DEBUG [Thread-1570 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T03:18:01,230 DEBUG [Thread-1570 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:18:01,231 DEBUG [Thread-1570 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:18:01,231 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T03:18:01,232 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T03:18:01,235 INFO [Thread-1566 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1566. 2024-11-10T03:18:01,235 DEBUG [Thread-1566 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T03:18:01,235 DEBUG [Thread-1566 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:18:01,235 DEBUG [Thread-1566 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:18:01,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742269_1445 (size=32107) 2024-11-10T03:18:01,238 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T03:18:01,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110fab31cf42e1c4470818086f99f8aff01_c6879c5307a3686e6731f46d96d3ca08 is 50, key is test_row_0/A:col10/1731208679778/Put/seqid=0 2024-11-10T03:18:01,242 INFO [Thread-1568 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1568. 2024-11-10T03:18:01,243 DEBUG [Thread-1568 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T03:18:01,243 DEBUG [Thread-1568 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:18:01,243 DEBUG [Thread-1568 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:18:01,243 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T03:18:01,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742270_1446 (size=12454) 2024-11-10T03:18:01,254 INFO [Thread-1564 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1564. 2024-11-10T03:18:01,254 DEBUG [Thread-1564 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T03:18:01,254 DEBUG [Thread-1564 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:18:01,254 DEBUG [Thread-1564 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:18:01,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:18:01,256 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T03:18:01,260 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110fab31cf42e1c4470818086f99f8aff01_c6879c5307a3686e6731f46d96d3ca08 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110fab31cf42e1c4470818086f99f8aff01_c6879c5307a3686e6731f46d96d3ca08 2024-11-10T03:18:01,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/A/312a04b4bf874f7e832e29558f000caa, store: [table=TestAcidGuarantees family=A region=c6879c5307a3686e6731f46d96d3ca08] 2024-11-10T03:18:01,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/A/312a04b4bf874f7e832e29558f000caa is 175, key is test_row_0/A:col10/1731208679778/Put/seqid=0 2024-11-10T03:18:01,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742271_1447 (size=31255) 2024-11-10T03:18:01,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-10T03:18:01,603 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/C/7db09b02ba284dba87b479b4a931011b as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/7db09b02ba284dba87b479b4a931011b 2024-11-10T03:18:01,616 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c6879c5307a3686e6731f46d96d3ca08/C of c6879c5307a3686e6731f46d96d3ca08 into 7db09b02ba284dba87b479b4a931011b(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:18:01,616 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c6879c5307a3686e6731f46d96d3ca08: 2024-11-10T03:18:01,616 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., storeName=c6879c5307a3686e6731f46d96d3ca08/C, priority=13, startTime=1731208681101; duration=0sec 2024-11-10T03:18:01,616 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T03:18:01,616 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c6879c5307a3686e6731f46d96d3ca08:C 2024-11-10T03:18:01,616 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T03:18:01,617 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T03:18:01,617 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1541): c6879c5307a3686e6731f46d96d3ca08/B is initiating minor compaction (all files) 2024-11-10T03:18:01,617 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c6879c5307a3686e6731f46d96d3ca08/B in TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:18:01,617 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/0f1efa95b009435882d6333a91a5c135, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/c8a4ff9d7aba4f179d8cfa61b4aa30af, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/d357b34481bf4112bbf347c551c51798] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp, totalSize=36.8 K 2024-11-10T03:18:01,618 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f1efa95b009435882d6333a91a5c135, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1731208677908 2024-11-10T03:18:01,618 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting c8a4ff9d7aba4f179d8cfa61b4aa30af, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1731208678076 2024-11-10T03:18:01,618 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting d357b34481bf4112bbf347c551c51798, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1731208679305 2024-11-10T03:18:01,626 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c6879c5307a3686e6731f46d96d3ca08#B#compaction#369 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:18:01,626 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/B/363ad7c38e7d4fc9870faa246aa94a77 is 50, key is test_row_0/B:col10/1731208679673/Put/seqid=0 2024-11-10T03:18:01,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742272_1448 (size=13153) 2024-11-10T03:18:01,646 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/A/f467615b58ad463782ddf3915cd5ec72 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/f467615b58ad463782ddf3915cd5ec72 2024-11-10T03:18:01,653 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c6879c5307a3686e6731f46d96d3ca08/A of c6879c5307a3686e6731f46d96d3ca08 into f467615b58ad463782ddf3915cd5ec72(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:18:01,653 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c6879c5307a3686e6731f46d96d3ca08: 2024-11-10T03:18:01,653 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., storeName=c6879c5307a3686e6731f46d96d3ca08/A, priority=13, startTime=1731208681101; duration=0sec 2024-11-10T03:18:01,653 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:18:01,653 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c6879c5307a3686e6731f46d96d3ca08:A 2024-11-10T03:18:01,657 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/B/363ad7c38e7d4fc9870faa246aa94a77 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/363ad7c38e7d4fc9870faa246aa94a77 2024-11-10T03:18:01,664 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c6879c5307a3686e6731f46d96d3ca08/B of c6879c5307a3686e6731f46d96d3ca08 into 363ad7c38e7d4fc9870faa246aa94a77(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:18:01,664 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c6879c5307a3686e6731f46d96d3ca08: 2024-11-10T03:18:01,664 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., storeName=c6879c5307a3686e6731f46d96d3ca08/B, priority=13, startTime=1731208681101; duration=0sec 2024-11-10T03:18:01,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on c6879c5307a3686e6731f46d96d3ca08 2024-11-10T03:18:01,664 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:18:01,664 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c6879c5307a3686e6731f46d96d3ca08:B 2024-11-10T03:18:01,664 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. as already flushing 2024-11-10T03:18:01,665 INFO [Thread-1561 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1561. 2024-11-10T03:18:01,665 DEBUG [Thread-1561 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T03:18:01,665 DEBUG [Thread-1561 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:18:01,665 DEBUG [Thread-1561 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:18:01,665 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T03:18:01,682 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=389, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/A/312a04b4bf874f7e832e29558f000caa 2024-11-10T03:18:01,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/B/41f1459af4f44286bb154b36f700e6c2 is 50, key is test_row_0/B:col10/1731208679778/Put/seqid=0 2024-11-10T03:18:01,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742273_1449 (size=12301) 2024-11-10T03:18:01,954 INFO [Thread-1555 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1555. 2024-11-10T03:18:01,954 DEBUG [Thread-1555 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T03:18:01,955 DEBUG [Thread-1555 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:18:01,955 DEBUG [Thread-1555 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:18:01,955 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T03:18:01,965 INFO [Thread-1559 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1559. 2024-11-10T03:18:01,965 DEBUG [Thread-1559 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T03:18:01,965 DEBUG [Thread-1559 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:18:01,965 DEBUG [Thread-1559 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:18:01,965 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T03:18:01,974 INFO [Thread-1557 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1557. 2024-11-10T03:18:01,974 DEBUG [Thread-1557 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T03:18:01,974 DEBUG [Thread-1557 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:18:01,974 DEBUG [Thread-1557 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:18:01,974 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T03:18:02,094 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=389 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/B/41f1459af4f44286bb154b36f700e6c2 2024-11-10T03:18:02,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/C/f78d01ff7790457fbdd2611692ab565b is 50, key is test_row_0/C:col10/1731208679778/Put/seqid=0 2024-11-10T03:18:02,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742274_1450 (size=12301) 2024-11-10T03:18:02,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-10T03:18:02,505 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=389 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/C/f78d01ff7790457fbdd2611692ab565b 2024-11-10T03:18:02,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/A/312a04b4bf874f7e832e29558f000caa as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/312a04b4bf874f7e832e29558f000caa 2024-11-10T03:18:02,512 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/312a04b4bf874f7e832e29558f000caa, entries=150, sequenceid=389, filesize=30.5 K 2024-11-10T03:18:02,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/B/41f1459af4f44286bb154b36f700e6c2 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/41f1459af4f44286bb154b36f700e6c2 2024-11-10T03:18:02,515 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/41f1459af4f44286bb154b36f700e6c2, entries=150, sequenceid=389, filesize=12.0 K 2024-11-10T03:18:02,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/C/f78d01ff7790457fbdd2611692ab565b as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/f78d01ff7790457fbdd2611692ab565b 2024-11-10T03:18:02,518 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/f78d01ff7790457fbdd2611692ab565b, entries=150, sequenceid=389, filesize=12.0 K 2024-11-10T03:18:02,519 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=26.84 KB/27480 for c6879c5307a3686e6731f46d96d3ca08 in 1310ms, sequenceid=389, compaction requested=false 2024-11-10T03:18:02,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for c6879c5307a3686e6731f46d96d3ca08: 2024-11-10T03:18:02,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:18:02,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-10T03:18:02,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-10T03:18:02,521 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-10T03:18:02,522 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2570 sec 2024-11-10T03:18:02,523 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 2.2680 sec 2024-11-10T03:18:04,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-10T03:18:04,443 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T03:18:09,846 INFO [Thread-1553 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1553. 2024-11-10T03:18:09,846 DEBUG [Thread-1553 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T03:18:09,846 DEBUG [Thread-1553 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:18:09,846 DEBUG [Thread-1553 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:18:09,846 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T03:18:09,846 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-10T03:18:09,847 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 51 2024-11-10T03:18:09,847 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 81 2024-11-10T03:18:09,847 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 78 2024-11-10T03:18:09,847 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 90 2024-11-10T03:18:09,847 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 12 2024-11-10T03:18:09,847 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-10T03:18:09,847 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-10T03:18:09,847 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 800 2024-11-10T03:18:09,847 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2392 rows 2024-11-10T03:18:09,847 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 798 2024-11-10T03:18:09,847 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2388 rows 2024-11-10T03:18:09,847 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 799 2024-11-10T03:18:09,847 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2386 rows 2024-11-10T03:18:09,847 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 797 2024-11-10T03:18:09,847 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2389 rows 2024-11-10T03:18:09,847 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 797 2024-11-10T03:18:09,847 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2387 rows 2024-11-10T03:18:09,847 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T03:18:09,847 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T03:18:09,847 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:18:09,848 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:18:09,848 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T03:18:09,849 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-10T03:18:09,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-10T03:18:09,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-10T03:18:09,853 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731208689853"}]},"ts":"1731208689853"} 2024-11-10T03:18:09,855 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-10T03:18:09,855 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-10T03:18:09,857 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-10T03:18:09,859 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c6879c5307a3686e6731f46d96d3ca08, UNASSIGN}] 2024-11-10T03:18:09,860 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c6879c5307a3686e6731f46d96d3ca08, UNASSIGN 2024-11-10T03:18:09,861 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=111 updating hbase:meta row=c6879c5307a3686e6731f46d96d3ca08, regionState=CLOSING, regionLocation=394d2222f724,38139,1731208572054 2024-11-10T03:18:09,863 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c6879c5307a3686e6731f46d96d3ca08, UNASSIGN because future has completed 2024-11-10T03:18:09,864 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-10T03:18:09,864 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; CloseRegionProcedure c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054}] 2024-11-10T03:18:09,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-10T03:18:10,017 INFO [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] handler.UnassignRegionHandler(122): Close c6879c5307a3686e6731f46d96d3ca08 2024-11-10T03:18:10,017 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-10T03:18:10,017 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1722): Closing c6879c5307a3686e6731f46d96d3ca08, disabling compactions & flushes 2024-11-10T03:18:10,017 INFO [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:18:10,017 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:18:10,017 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. after waiting 0 ms 2024-11-10T03:18:10,017 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:18:10,017 INFO [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(2902): Flushing c6879c5307a3686e6731f46d96d3ca08 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-10T03:18:10,017 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6879c5307a3686e6731f46d96d3ca08, store=A 2024-11-10T03:18:10,017 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:18:10,017 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6879c5307a3686e6731f46d96d3ca08, store=B 2024-11-10T03:18:10,017 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:18:10,017 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6879c5307a3686e6731f46d96d3ca08, store=C 2024-11-10T03:18:10,018 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:18:10,022 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111076e06daf5e374f589c3c5f8010f05a29_c6879c5307a3686e6731f46d96d3ca08 is 50, key is test_row_0/A:col10/1731208681964/Put/seqid=0 2024-11-10T03:18:10,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742275_1451 (size=9914) 2024-11-10T03:18:10,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-10T03:18:10,311 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-10T03:18:10,427 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:18:10,430 INFO [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111076e06daf5e374f589c3c5f8010f05a29_c6879c5307a3686e6731f46d96d3ca08 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111076e06daf5e374f589c3c5f8010f05a29_c6879c5307a3686e6731f46d96d3ca08 2024-11-10T03:18:10,431 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/A/95ce0e3ae5024fbb839a13b9fd0279e7, store: [table=TestAcidGuarantees family=A region=c6879c5307a3686e6731f46d96d3ca08] 2024-11-10T03:18:10,431 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/A/95ce0e3ae5024fbb839a13b9fd0279e7 is 175, key is test_row_0/A:col10/1731208681964/Put/seqid=0 2024-11-10T03:18:10,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742276_1452 (size=22561) 2024-11-10T03:18:10,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-10T03:18:10,835 INFO [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=400, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/A/95ce0e3ae5024fbb839a13b9fd0279e7 2024-11-10T03:18:10,840 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/B/0ad34afad8d544b79b7664adb2bcee18 is 50, key is test_row_0/B:col10/1731208681964/Put/seqid=0 2024-11-10T03:18:10,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742277_1453 (size=9857) 2024-11-10T03:18:10,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-10T03:18:11,243 INFO [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=400 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/B/0ad34afad8d544b79b7664adb2bcee18 2024-11-10T03:18:11,249 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/C/346a5446120c4ec89b1d435213651b75 is 50, key is test_row_0/C:col10/1731208681964/Put/seqid=0 2024-11-10T03:18:11,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742278_1454 (size=9857) 2024-11-10T03:18:11,653 INFO [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=400 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/C/346a5446120c4ec89b1d435213651b75 2024-11-10T03:18:11,657 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/A/95ce0e3ae5024fbb839a13b9fd0279e7 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/95ce0e3ae5024fbb839a13b9fd0279e7 2024-11-10T03:18:11,660 INFO [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/95ce0e3ae5024fbb839a13b9fd0279e7, entries=100, sequenceid=400, filesize=22.0 K 2024-11-10T03:18:11,660 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/B/0ad34afad8d544b79b7664adb2bcee18 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/0ad34afad8d544b79b7664adb2bcee18 2024-11-10T03:18:11,663 INFO [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/0ad34afad8d544b79b7664adb2bcee18, entries=100, sequenceid=400, filesize=9.6 K 2024-11-10T03:18:11,664 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/.tmp/C/346a5446120c4ec89b1d435213651b75 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/346a5446120c4ec89b1d435213651b75 2024-11-10T03:18:11,666 INFO [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/346a5446120c4ec89b1d435213651b75, entries=100, sequenceid=400, filesize=9.6 K 2024-11-10T03:18:11,667 INFO [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for c6879c5307a3686e6731f46d96d3ca08 in 1650ms, sequenceid=400, compaction requested=true 2024-11-10T03:18:11,667 DEBUG [StoreCloser-TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/7fac2dd553654faa9bd537ed3e8b83cc, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/842e6f66ebce4bbdab1d5854de7cc815, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/06b454b931ba487ebac5af1c649e1a8e, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/b432b2aa53a44f9bb188b32cb5d53dae, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/f04ea5eb8d324cc9a854b752d1a0fcdd, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/6a0fdfac12404b6f9913d0e9a3878427, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/31582fe8a28a4840aa7bc430a4b44ce8, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/9d701ae590b2482aac0ee6fcdb89bac9, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/c3eeab4d778b4e47a44ae13b2bad554f, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/7b32c100c9014b25b774f2f43df55a33, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/6888815fd6f94a2993960b8ca125b139, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/3d18f839f37a4319b454f86617be30b5, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/151470833e7c4978a450830a8a879aa5, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/195069d59f334396bc408f1c793cebbf, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/1129c49e7cd4436684186c32375c56d0, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/1405fa84825d4ceeb14587e42be94948, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/87d156ff19d44e99b2ee2df8e93ccaa3, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/9afd9eff4e5d4caf8726330b3c2af5db, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/c9f4223f5fc04c84bb5bd0212162038c, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/27338101b7924542a003a29221afc034, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/05b07ee73b7541b686d8b1aafe726c0e, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/473fca4c8c6f48f5ba97dcf6289fe18b, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/38b601ce04ae4ac89c3acba38095611a, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/ece281eddcd5458890469941757c5ad6, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/66d46b69b9124f80abe859f9e1ce41d1] to archive 2024-11-10T03:18:11,668 DEBUG [StoreCloser-TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T03:18:11,670 DEBUG [StoreCloser-TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/7fac2dd553654faa9bd537ed3e8b83cc to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/7fac2dd553654faa9bd537ed3e8b83cc 2024-11-10T03:18:11,670 DEBUG [StoreCloser-TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/842e6f66ebce4bbdab1d5854de7cc815 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/842e6f66ebce4bbdab1d5854de7cc815 2024-11-10T03:18:11,671 DEBUG [StoreCloser-TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/06b454b931ba487ebac5af1c649e1a8e to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/06b454b931ba487ebac5af1c649e1a8e 2024-11-10T03:18:11,672 DEBUG [StoreCloser-TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/b432b2aa53a44f9bb188b32cb5d53dae to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/b432b2aa53a44f9bb188b32cb5d53dae 2024-11-10T03:18:11,673 DEBUG [StoreCloser-TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/f04ea5eb8d324cc9a854b752d1a0fcdd to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/f04ea5eb8d324cc9a854b752d1a0fcdd 2024-11-10T03:18:11,675 DEBUG [StoreCloser-TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/6a0fdfac12404b6f9913d0e9a3878427 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/6a0fdfac12404b6f9913d0e9a3878427 2024-11-10T03:18:11,676 DEBUG [StoreCloser-TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/31582fe8a28a4840aa7bc430a4b44ce8 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/31582fe8a28a4840aa7bc430a4b44ce8 2024-11-10T03:18:11,677 DEBUG [StoreCloser-TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/9d701ae590b2482aac0ee6fcdb89bac9 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/9d701ae590b2482aac0ee6fcdb89bac9 2024-11-10T03:18:11,678 DEBUG [StoreCloser-TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/c3eeab4d778b4e47a44ae13b2bad554f to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/c3eeab4d778b4e47a44ae13b2bad554f 2024-11-10T03:18:11,679 DEBUG [StoreCloser-TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/7b32c100c9014b25b774f2f43df55a33 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/7b32c100c9014b25b774f2f43df55a33 2024-11-10T03:18:11,680 DEBUG [StoreCloser-TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/6888815fd6f94a2993960b8ca125b139 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/6888815fd6f94a2993960b8ca125b139 2024-11-10T03:18:11,682 DEBUG [StoreCloser-TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/3d18f839f37a4319b454f86617be30b5 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/3d18f839f37a4319b454f86617be30b5 2024-11-10T03:18:11,683 DEBUG [StoreCloser-TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/151470833e7c4978a450830a8a879aa5 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/151470833e7c4978a450830a8a879aa5 2024-11-10T03:18:11,684 DEBUG [StoreCloser-TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/195069d59f334396bc408f1c793cebbf to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/195069d59f334396bc408f1c793cebbf 2024-11-10T03:18:11,685 DEBUG [StoreCloser-TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/1129c49e7cd4436684186c32375c56d0 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/1129c49e7cd4436684186c32375c56d0 2024-11-10T03:18:11,686 DEBUG [StoreCloser-TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/1405fa84825d4ceeb14587e42be94948 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/1405fa84825d4ceeb14587e42be94948 2024-11-10T03:18:11,687 DEBUG [StoreCloser-TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/87d156ff19d44e99b2ee2df8e93ccaa3 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/87d156ff19d44e99b2ee2df8e93ccaa3 2024-11-10T03:18:11,688 DEBUG [StoreCloser-TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/9afd9eff4e5d4caf8726330b3c2af5db to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/9afd9eff4e5d4caf8726330b3c2af5db 2024-11-10T03:18:11,689 DEBUG [StoreCloser-TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/c9f4223f5fc04c84bb5bd0212162038c to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/c9f4223f5fc04c84bb5bd0212162038c 2024-11-10T03:18:11,690 DEBUG [StoreCloser-TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/27338101b7924542a003a29221afc034 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/27338101b7924542a003a29221afc034 2024-11-10T03:18:11,690 DEBUG [StoreCloser-TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/05b07ee73b7541b686d8b1aafe726c0e to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/05b07ee73b7541b686d8b1aafe726c0e 2024-11-10T03:18:11,691 DEBUG [StoreCloser-TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/473fca4c8c6f48f5ba97dcf6289fe18b to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/473fca4c8c6f48f5ba97dcf6289fe18b 2024-11-10T03:18:11,692 DEBUG [StoreCloser-TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/38b601ce04ae4ac89c3acba38095611a to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/38b601ce04ae4ac89c3acba38095611a 2024-11-10T03:18:11,693 DEBUG [StoreCloser-TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/ece281eddcd5458890469941757c5ad6 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/ece281eddcd5458890469941757c5ad6 2024-11-10T03:18:11,694 DEBUG [StoreCloser-TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/66d46b69b9124f80abe859f9e1ce41d1 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/66d46b69b9124f80abe859f9e1ce41d1 2024-11-10T03:18:11,695 DEBUG [StoreCloser-TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/df5dec3c8cf34bc3a636f7880cf6e713, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/dec5abd1b1174959a025f5d883df45f9, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/92cb5392a2eb4b88bacf4d0f8a14b538, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/4ee332dae6854afe97843e4d8b72c2ff, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/fdebc8214ddb447493f904b4773c1d0c, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/20c205c8ee5d4744b44565541af3cba8, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/ac5d3b6f752645aaa8e433a0d1b627d0, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/11e071d35d634d16892467fd4fdd73d3, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/fbc20d0410d54d96b6547254267e500c, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/9c00d5e46bf94f218f452829f3cf2ef9, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/6570da8fc3ce4670ad83f27ff6ecb01d, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/21db582b735241f4811ea5e2bbb0c5b6, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/d116045cd9144320975e85b8ad507879, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/8e38bf8df49a4e3da25345fb2993fa16, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/1f5c6b9ee4e544509063b6f91c65bc59, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/cf91029945ef4b14bf41dc548b555934, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/cfccbadf71214c25ba968465be75bffe, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/45db82e440dc46ffa3aeaae0cd957e71, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/22ad3ae2088745658a17fc0386b650ec, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/92eea1336ddf4297b9796d2693a87c46, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/8ca50efbc7a5419eb0705f36601f9252, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/0f1efa95b009435882d6333a91a5c135, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/03ceb841cc814322b2ce6bb1482fdab1, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/c8a4ff9d7aba4f179d8cfa61b4aa30af, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/d357b34481bf4112bbf347c551c51798] to archive 2024-11-10T03:18:11,697 DEBUG [StoreCloser-TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T03:18:11,699 DEBUG [StoreCloser-TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/df5dec3c8cf34bc3a636f7880cf6e713 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/df5dec3c8cf34bc3a636f7880cf6e713 2024-11-10T03:18:11,700 DEBUG [StoreCloser-TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/dec5abd1b1174959a025f5d883df45f9 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/dec5abd1b1174959a025f5d883df45f9 2024-11-10T03:18:11,701 DEBUG [StoreCloser-TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/92cb5392a2eb4b88bacf4d0f8a14b538 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/92cb5392a2eb4b88bacf4d0f8a14b538 2024-11-10T03:18:11,703 DEBUG [StoreCloser-TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/4ee332dae6854afe97843e4d8b72c2ff to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/4ee332dae6854afe97843e4d8b72c2ff 2024-11-10T03:18:11,704 DEBUG [StoreCloser-TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/fdebc8214ddb447493f904b4773c1d0c to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/fdebc8214ddb447493f904b4773c1d0c 2024-11-10T03:18:11,705 DEBUG [StoreCloser-TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/20c205c8ee5d4744b44565541af3cba8 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/20c205c8ee5d4744b44565541af3cba8 2024-11-10T03:18:11,706 DEBUG [StoreCloser-TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/ac5d3b6f752645aaa8e433a0d1b627d0 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/ac5d3b6f752645aaa8e433a0d1b627d0 2024-11-10T03:18:11,707 DEBUG [StoreCloser-TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/11e071d35d634d16892467fd4fdd73d3 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/11e071d35d634d16892467fd4fdd73d3 2024-11-10T03:18:11,708 DEBUG [StoreCloser-TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/fbc20d0410d54d96b6547254267e500c to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/fbc20d0410d54d96b6547254267e500c 2024-11-10T03:18:11,709 DEBUG [StoreCloser-TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/9c00d5e46bf94f218f452829f3cf2ef9 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/9c00d5e46bf94f218f452829f3cf2ef9 2024-11-10T03:18:11,710 DEBUG [StoreCloser-TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/6570da8fc3ce4670ad83f27ff6ecb01d to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/6570da8fc3ce4670ad83f27ff6ecb01d 2024-11-10T03:18:11,710 DEBUG [StoreCloser-TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/21db582b735241f4811ea5e2bbb0c5b6 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/21db582b735241f4811ea5e2bbb0c5b6 2024-11-10T03:18:11,711 DEBUG [StoreCloser-TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/d116045cd9144320975e85b8ad507879 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/d116045cd9144320975e85b8ad507879 2024-11-10T03:18:11,712 DEBUG [StoreCloser-TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/8e38bf8df49a4e3da25345fb2993fa16 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/8e38bf8df49a4e3da25345fb2993fa16 2024-11-10T03:18:11,713 DEBUG [StoreCloser-TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/1f5c6b9ee4e544509063b6f91c65bc59 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/1f5c6b9ee4e544509063b6f91c65bc59 2024-11-10T03:18:11,714 DEBUG [StoreCloser-TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/cf91029945ef4b14bf41dc548b555934 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/cf91029945ef4b14bf41dc548b555934 2024-11-10T03:18:11,715 DEBUG [StoreCloser-TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/cfccbadf71214c25ba968465be75bffe to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/cfccbadf71214c25ba968465be75bffe 2024-11-10T03:18:11,715 DEBUG [StoreCloser-TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/45db82e440dc46ffa3aeaae0cd957e71 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/45db82e440dc46ffa3aeaae0cd957e71 2024-11-10T03:18:11,718 DEBUG [StoreCloser-TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/22ad3ae2088745658a17fc0386b650ec to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/22ad3ae2088745658a17fc0386b650ec 2024-11-10T03:18:11,719 DEBUG [StoreCloser-TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/92eea1336ddf4297b9796d2693a87c46 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/92eea1336ddf4297b9796d2693a87c46 2024-11-10T03:18:11,720 DEBUG [StoreCloser-TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/8ca50efbc7a5419eb0705f36601f9252 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/8ca50efbc7a5419eb0705f36601f9252 2024-11-10T03:18:11,721 DEBUG [StoreCloser-TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/0f1efa95b009435882d6333a91a5c135 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/0f1efa95b009435882d6333a91a5c135 2024-11-10T03:18:11,722 DEBUG [StoreCloser-TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/03ceb841cc814322b2ce6bb1482fdab1 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/03ceb841cc814322b2ce6bb1482fdab1 2024-11-10T03:18:11,723 DEBUG [StoreCloser-TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/c8a4ff9d7aba4f179d8cfa61b4aa30af to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/c8a4ff9d7aba4f179d8cfa61b4aa30af 2024-11-10T03:18:11,724 DEBUG [StoreCloser-TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/d357b34481bf4112bbf347c551c51798 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/d357b34481bf4112bbf347c551c51798 2024-11-10T03:18:11,725 DEBUG [StoreCloser-TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/bbb0253fa2a949e7b4b6af00dab1930b, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/6ca0ea000c5049138f13440cbf77ea5b, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/61048f5ffb7e4985a77c30b50d2a0cb6, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/5461ea9806454b5e9d1893bb55467690, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/73161c86224746bba1de7c1fa2a61b8d, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/d27ca857e4b9450a952d0dd2a096db74, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/6a2ee4fda36f443ea25a79724d3b17bd, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/dc1026052638488986a9580a91a04fc4, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/893cbf97a9fc4cb0a37a373db52ab29f, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/d33f1ad4b2b94214a61cb6e41689b463, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/bfb14d1aafa24f96973327c4d2720ef1, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/572b08420f50472dbb97cdf1667c4bb6, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/ece0b4c6165448e990025df7b77e00ad, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/fdfeda849bc74df286979cf33924ef3b, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/a401c90232b841e8badc606f7378f0bb, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/ce67747827d54eb8bbc394449199d798, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/4a9152e5371d4d0fb00ec7bbf1a332a7, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/65b40767936e47d6901cdc1fa203155c, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/c503f34ca2844c40bd21884a3c31a5af, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/40a8e01cbe724da798e74c1d8a0fd9e0, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/31939a83d581466fb9710a7fcf69daf5, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/06fa856c30f043b7b5d7bb26bc88b40c, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/b1371b51bcc245c8a4d8768bafbe5a1b, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/df6a368c4f5c4a68b57e43513ff8fdca, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/93651eb5b276480a89eb5d14e4377e3a] to archive 2024-11-10T03:18:11,726 DEBUG [StoreCloser-TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T03:18:11,727 DEBUG [StoreCloser-TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/bbb0253fa2a949e7b4b6af00dab1930b to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/bbb0253fa2a949e7b4b6af00dab1930b 2024-11-10T03:18:11,728 DEBUG [StoreCloser-TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/6ca0ea000c5049138f13440cbf77ea5b to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/6ca0ea000c5049138f13440cbf77ea5b 2024-11-10T03:18:11,729 DEBUG [StoreCloser-TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/61048f5ffb7e4985a77c30b50d2a0cb6 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/61048f5ffb7e4985a77c30b50d2a0cb6 2024-11-10T03:18:11,730 DEBUG [StoreCloser-TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/5461ea9806454b5e9d1893bb55467690 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/5461ea9806454b5e9d1893bb55467690 2024-11-10T03:18:11,731 DEBUG [StoreCloser-TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/73161c86224746bba1de7c1fa2a61b8d to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/73161c86224746bba1de7c1fa2a61b8d 2024-11-10T03:18:11,732 DEBUG [StoreCloser-TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/d27ca857e4b9450a952d0dd2a096db74 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/d27ca857e4b9450a952d0dd2a096db74 2024-11-10T03:18:11,733 DEBUG [StoreCloser-TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/6a2ee4fda36f443ea25a79724d3b17bd to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/6a2ee4fda36f443ea25a79724d3b17bd 2024-11-10T03:18:11,734 DEBUG [StoreCloser-TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/dc1026052638488986a9580a91a04fc4 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/dc1026052638488986a9580a91a04fc4 2024-11-10T03:18:11,735 DEBUG [StoreCloser-TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/893cbf97a9fc4cb0a37a373db52ab29f to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/893cbf97a9fc4cb0a37a373db52ab29f 2024-11-10T03:18:11,736 DEBUG [StoreCloser-TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/d33f1ad4b2b94214a61cb6e41689b463 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/d33f1ad4b2b94214a61cb6e41689b463 2024-11-10T03:18:11,737 DEBUG [StoreCloser-TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/bfb14d1aafa24f96973327c4d2720ef1 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/bfb14d1aafa24f96973327c4d2720ef1 2024-11-10T03:18:11,737 DEBUG [StoreCloser-TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/572b08420f50472dbb97cdf1667c4bb6 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/572b08420f50472dbb97cdf1667c4bb6 2024-11-10T03:18:11,738 DEBUG [StoreCloser-TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/ece0b4c6165448e990025df7b77e00ad to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/ece0b4c6165448e990025df7b77e00ad 2024-11-10T03:18:11,739 DEBUG [StoreCloser-TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/fdfeda849bc74df286979cf33924ef3b to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/fdfeda849bc74df286979cf33924ef3b 2024-11-10T03:18:11,740 DEBUG [StoreCloser-TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/a401c90232b841e8badc606f7378f0bb to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/a401c90232b841e8badc606f7378f0bb 2024-11-10T03:18:11,741 DEBUG [StoreCloser-TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/ce67747827d54eb8bbc394449199d798 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/ce67747827d54eb8bbc394449199d798 2024-11-10T03:18:11,742 DEBUG [StoreCloser-TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/4a9152e5371d4d0fb00ec7bbf1a332a7 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/4a9152e5371d4d0fb00ec7bbf1a332a7 2024-11-10T03:18:11,743 DEBUG [StoreCloser-TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/65b40767936e47d6901cdc1fa203155c to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/65b40767936e47d6901cdc1fa203155c 2024-11-10T03:18:11,744 DEBUG [StoreCloser-TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/c503f34ca2844c40bd21884a3c31a5af to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/c503f34ca2844c40bd21884a3c31a5af 2024-11-10T03:18:11,745 DEBUG [StoreCloser-TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/40a8e01cbe724da798e74c1d8a0fd9e0 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/40a8e01cbe724da798e74c1d8a0fd9e0 2024-11-10T03:18:11,746 DEBUG [StoreCloser-TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/31939a83d581466fb9710a7fcf69daf5 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/31939a83d581466fb9710a7fcf69daf5 2024-11-10T03:18:11,746 DEBUG [StoreCloser-TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/06fa856c30f043b7b5d7bb26bc88b40c to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/06fa856c30f043b7b5d7bb26bc88b40c 2024-11-10T03:18:11,747 DEBUG [StoreCloser-TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/b1371b51bcc245c8a4d8768bafbe5a1b to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/b1371b51bcc245c8a4d8768bafbe5a1b 2024-11-10T03:18:11,748 DEBUG [StoreCloser-TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/df6a368c4f5c4a68b57e43513ff8fdca to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/df6a368c4f5c4a68b57e43513ff8fdca 2024-11-10T03:18:11,749 DEBUG [StoreCloser-TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/93651eb5b276480a89eb5d14e4377e3a to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/93651eb5b276480a89eb5d14e4377e3a 2024-11-10T03:18:11,752 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/recovered.edits/403.seqid, newMaxSeqId=403, maxSeqId=4 2024-11-10T03:18:11,752 INFO [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. 2024-11-10T03:18:11,752 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1676): Region close journal for c6879c5307a3686e6731f46d96d3ca08: Waiting for close lock at 1731208690017Running coprocessor pre-close hooks at 1731208690017Disabling compacts and flushes for region at 1731208690017Disabling writes for close at 1731208690017Obtaining lock to block concurrent updates at 1731208690017Preparing flush snapshotting stores in c6879c5307a3686e6731f46d96d3ca08 at 1731208690017Finished memstore snapshotting TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731208690018 (+1 ms)Flushing stores of TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08. at 1731208690018Flushing c6879c5307a3686e6731f46d96d3ca08/A: creating writer at 1731208690018Flushing mob file c6879c5307a3686e6731f46d96d3ca08/A: appending metadata at 1731208690022 (+4 ms)Flushing mob file c6879c5307a3686e6731f46d96d3ca08/A: closing flushed file at 1731208690022Flushing c6879c5307a3686e6731f46d96d3ca08/A: appending metadata at 1731208690431 (+409 ms)Flushing c6879c5307a3686e6731f46d96d3ca08/A: closing flushed file at 1731208690431Flushing c6879c5307a3686e6731f46d96d3ca08/B: creating writer at 1731208690837 (+406 ms)Flushing c6879c5307a3686e6731f46d96d3ca08/B: appending metadata at 1731208690839 (+2 ms)Flushing c6879c5307a3686e6731f46d96d3ca08/B: closing flushed file at 1731208690840 (+1 ms)Flushing c6879c5307a3686e6731f46d96d3ca08/C: creating writer at 1731208691246 (+406 ms)Flushing c6879c5307a3686e6731f46d96d3ca08/C: appending metadata at 1731208691249 (+3 ms)Flushing c6879c5307a3686e6731f46d96d3ca08/C: closing flushed file at 1731208691249Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7d0dbf48: reopening flushed file at 1731208691656 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7e986732: reopening flushed file at 1731208691660 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3524ecbf: reopening flushed file at 1731208691663 (+3 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for c6879c5307a3686e6731f46d96d3ca08 in 1650ms, sequenceid=400, compaction requested=true at 1731208691667 (+4 ms)Writing region close event to WAL at 1731208691750 (+83 ms)Running coprocessor post-close hooks at 1731208691752 (+2 ms)Closed at 1731208691752 2024-11-10T03:18:11,754 INFO [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] handler.UnassignRegionHandler(157): Closed c6879c5307a3686e6731f46d96d3ca08 2024-11-10T03:18:11,754 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=111 updating hbase:meta row=c6879c5307a3686e6731f46d96d3ca08, regionState=CLOSED 2024-11-10T03:18:11,755 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=112, ppid=111, state=RUNNABLE, hasLock=false; CloseRegionProcedure c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 because future has completed 2024-11-10T03:18:11,757 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-10T03:18:11,757 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; CloseRegionProcedure c6879c5307a3686e6731f46d96d3ca08, server=394d2222f724,38139,1731208572054 in 1.8920 sec 2024-11-10T03:18:11,759 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=111, resume processing ppid=110 2024-11-10T03:18:11,759 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, ppid=110, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c6879c5307a3686e6731f46d96d3ca08, UNASSIGN in 1.8980 sec 2024-11-10T03:18:11,761 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-10T03:18:11,761 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9030 sec 2024-11-10T03:18:11,762 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731208691762"}]},"ts":"1731208691762"} 2024-11-10T03:18:11,764 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-10T03:18:11,764 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-10T03:18:11,766 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9160 sec 2024-11-10T03:18:12,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-10T03:18:12,004 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-10T03:18:12,005 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-10T03:18:12,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T03:18:12,007 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=113, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T03:18:12,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-10T03:18:12,009 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=113, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T03:18:12,012 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08 2024-11-10T03:18:12,015 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A, FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B, FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C, FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/recovered.edits] 2024-11-10T03:18:12,019 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/312a04b4bf874f7e832e29558f000caa to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/312a04b4bf874f7e832e29558f000caa 2024-11-10T03:18:12,021 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/95ce0e3ae5024fbb839a13b9fd0279e7 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/95ce0e3ae5024fbb839a13b9fd0279e7 2024-11-10T03:18:12,023 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/f467615b58ad463782ddf3915cd5ec72 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/A/f467615b58ad463782ddf3915cd5ec72 2024-11-10T03:18:12,026 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/0ad34afad8d544b79b7664adb2bcee18 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/0ad34afad8d544b79b7664adb2bcee18 2024-11-10T03:18:12,028 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/363ad7c38e7d4fc9870faa246aa94a77 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/363ad7c38e7d4fc9870faa246aa94a77 2024-11-10T03:18:12,029 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/41f1459af4f44286bb154b36f700e6c2 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/B/41f1459af4f44286bb154b36f700e6c2 2024-11-10T03:18:12,033 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/346a5446120c4ec89b1d435213651b75 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/346a5446120c4ec89b1d435213651b75 2024-11-10T03:18:12,034 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/7db09b02ba284dba87b479b4a931011b to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/7db09b02ba284dba87b479b4a931011b 2024-11-10T03:18:12,035 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/f78d01ff7790457fbdd2611692ab565b to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/C/f78d01ff7790457fbdd2611692ab565b 2024-11-10T03:18:12,037 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/recovered.edits/403.seqid to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08/recovered.edits/403.seqid 2024-11-10T03:18:12,038 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/c6879c5307a3686e6731f46d96d3ca08 2024-11-10T03:18:12,038 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-10T03:18:12,038 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-10T03:18:12,038 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-10T03:18:12,041 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411100708b998fe7f40aa8355593a38c810ea_c6879c5307a3686e6731f46d96d3ca08 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411100708b998fe7f40aa8355593a38c810ea_c6879c5307a3686e6731f46d96d3ca08 2024-11-10T03:18:12,042 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411101ce9ed9570eb4ad0a777993a1a2fcf33_c6879c5307a3686e6731f46d96d3ca08 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411101ce9ed9570eb4ad0a777993a1a2fcf33_c6879c5307a3686e6731f46d96d3ca08 2024-11-10T03:18:12,042 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111021dd711d3c354386b2c5fe57452a943a_c6879c5307a3686e6731f46d96d3ca08 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111021dd711d3c354386b2c5fe57452a943a_c6879c5307a3686e6731f46d96d3ca08 2024-11-10T03:18:12,043 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411102f7b65edc96343de8af7009a16bae449_c6879c5307a3686e6731f46d96d3ca08 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411102f7b65edc96343de8af7009a16bae449_c6879c5307a3686e6731f46d96d3ca08 2024-11-10T03:18:12,044 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411103634af3a490f4565b90d69dae76cacfd_c6879c5307a3686e6731f46d96d3ca08 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411103634af3a490f4565b90d69dae76cacfd_c6879c5307a3686e6731f46d96d3ca08 2024-11-10T03:18:12,045 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111037d5aba0f003494d87a7d903c81428d7_c6879c5307a3686e6731f46d96d3ca08 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111037d5aba0f003494d87a7d903c81428d7_c6879c5307a3686e6731f46d96d3ca08 2024-11-10T03:18:12,046 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111039030f88612d4fa8bb4df8957fd173cf_c6879c5307a3686e6731f46d96d3ca08 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111039030f88612d4fa8bb4df8957fd173cf_c6879c5307a3686e6731f46d96d3ca08 2024-11-10T03:18:12,046 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411104a30ffbbeb854a859e7328b4cfdaf78c_c6879c5307a3686e6731f46d96d3ca08 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411104a30ffbbeb854a859e7328b4cfdaf78c_c6879c5307a3686e6731f46d96d3ca08 2024-11-10T03:18:12,047 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110540d8536d8144f8ca4d6bca4cfc0783f_c6879c5307a3686e6731f46d96d3ca08 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110540d8536d8144f8ca4d6bca4cfc0783f_c6879c5307a3686e6731f46d96d3ca08 2024-11-10T03:18:12,048 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111058e38dc638c04304bf60696506ddc0c9_c6879c5307a3686e6731f46d96d3ca08 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111058e38dc638c04304bf60696506ddc0c9_c6879c5307a3686e6731f46d96d3ca08 2024-11-10T03:18:12,048 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411106676d18bb2644359a4522ac9f9923ca9_c6879c5307a3686e6731f46d96d3ca08 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411106676d18bb2644359a4522ac9f9923ca9_c6879c5307a3686e6731f46d96d3ca08 2024-11-10T03:18:12,049 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411106b940f72035c4c2a864bffc2a76c46ab_c6879c5307a3686e6731f46d96d3ca08 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411106b940f72035c4c2a864bffc2a76c46ab_c6879c5307a3686e6731f46d96d3ca08 2024-11-10T03:18:12,050 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111076e06daf5e374f589c3c5f8010f05a29_c6879c5307a3686e6731f46d96d3ca08 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111076e06daf5e374f589c3c5f8010f05a29_c6879c5307a3686e6731f46d96d3ca08 2024-11-10T03:18:12,050 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110775f0a23d1d045e4832e8bacb37a9289_c6879c5307a3686e6731f46d96d3ca08 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110775f0a23d1d045e4832e8bacb37a9289_c6879c5307a3686e6731f46d96d3ca08 2024-11-10T03:18:12,051 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411109613421a4ffd47dcaa4a8aa274892325_c6879c5307a3686e6731f46d96d3ca08 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411109613421a4ffd47dcaa4a8aa274892325_c6879c5307a3686e6731f46d96d3ca08 2024-11-10T03:18:12,052 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110cb7c37c9280a4b048deb28eec40ca1cd_c6879c5307a3686e6731f46d96d3ca08 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110cb7c37c9280a4b048deb28eec40ca1cd_c6879c5307a3686e6731f46d96d3ca08 2024-11-10T03:18:12,053 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110ded3a2f9b34640dfac5978f316197b72_c6879c5307a3686e6731f46d96d3ca08 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110ded3a2f9b34640dfac5978f316197b72_c6879c5307a3686e6731f46d96d3ca08 2024-11-10T03:18:12,053 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110ecf133942456486993a2c3619f0d32cb_c6879c5307a3686e6731f46d96d3ca08 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110ecf133942456486993a2c3619f0d32cb_c6879c5307a3686e6731f46d96d3ca08 2024-11-10T03:18:12,054 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110f8e8929951cd4505acdff51acd208f8d_c6879c5307a3686e6731f46d96d3ca08 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110f8e8929951cd4505acdff51acd208f8d_c6879c5307a3686e6731f46d96d3ca08 2024-11-10T03:18:12,055 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110fab31cf42e1c4470818086f99f8aff01_c6879c5307a3686e6731f46d96d3ca08 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110fab31cf42e1c4470818086f99f8aff01_c6879c5307a3686e6731f46d96d3ca08 2024-11-10T03:18:12,055 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110fd2ee3e6a34742b18365d885fa819ef7_c6879c5307a3686e6731f46d96d3ca08 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110fd2ee3e6a34742b18365d885fa819ef7_c6879c5307a3686e6731f46d96d3ca08 2024-11-10T03:18:12,056 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-10T03:18:12,057 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=113, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T03:18:12,059 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-10T03:18:12,061 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-10T03:18:12,062 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=113, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T03:18:12,062 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-10T03:18:12,063 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731208692062"}]},"ts":"9223372036854775807"} 2024-11-10T03:18:12,065 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-10T03:18:12,065 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => c6879c5307a3686e6731f46d96d3ca08, NAME => 'TestAcidGuarantees,,1731208658488.c6879c5307a3686e6731f46d96d3ca08.', STARTKEY => '', ENDKEY => ''}] 2024-11-10T03:18:12,065 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-10T03:18:12,065 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731208692065"}]},"ts":"9223372036854775807"} 2024-11-10T03:18:12,066 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-10T03:18:12,067 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=113, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T03:18:12,068 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 62 msec 2024-11-10T03:18:12,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-10T03:18:12,113 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-10T03:18:12,114 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-10T03:18:12,123 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=247 (was 250), OpenFileDescriptor=451 (was 455), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=710 (was 690) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2111 (was 1009) - AvailableMemoryMB LEAK? - 2024-11-10T03:18:12,131 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=247, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=710, ProcessCount=11, AvailableMemoryMB=2111 2024-11-10T03:18:12,132 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-10T03:18:12,132 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T03:18:12,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] procedure2.ProcedureExecutor(1139): Stored pid=114, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-10T03:18:12,134 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-10T03:18:12,134 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:18:12,134 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 114 2024-11-10T03:18:12,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-11-10T03:18:12,135 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-10T03:18:12,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742279_1455 (size=1061) 2024-11-10T03:18:12,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-11-10T03:18:12,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-11-10T03:18:12,544 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8 2024-11-10T03:18:12,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742280_1456 (size=53) 2024-11-10T03:18:12,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-11-10T03:18:12,951 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T03:18:12,951 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 206af57796ad2b3d51883827ba0564a7, disabling compactions & flushes 2024-11-10T03:18:12,951 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:12,951 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:12,951 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. after waiting 0 ms 2024-11-10T03:18:12,951 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:12,951 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:12,951 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 206af57796ad2b3d51883827ba0564a7: Waiting for close lock at 1731208692951Disabling compacts and flushes for region at 1731208692951Disabling writes for close at 1731208692951Writing region close event to WAL at 1731208692951Closed at 1731208692951 2024-11-10T03:18:12,953 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-10T03:18:12,953 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731208692953"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731208692953"}]},"ts":"1731208692953"} 2024-11-10T03:18:12,954 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-10T03:18:12,955 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-10T03:18:12,955 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731208692955"}]},"ts":"1731208692955"} 2024-11-10T03:18:12,956 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-10T03:18:12,957 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=206af57796ad2b3d51883827ba0564a7, ASSIGN}] 2024-11-10T03:18:12,958 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=206af57796ad2b3d51883827ba0564a7, ASSIGN 2024-11-10T03:18:12,959 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=206af57796ad2b3d51883827ba0564a7, ASSIGN; state=OFFLINE, location=394d2222f724,38139,1731208572054; forceNewPlan=false, retain=false 2024-11-10T03:18:13,110 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=115 updating hbase:meta row=206af57796ad2b3d51883827ba0564a7, regionState=OPENING, regionLocation=394d2222f724,38139,1731208572054 2024-11-10T03:18:13,112 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=206af57796ad2b3d51883827ba0564a7, ASSIGN because future has completed 2024-11-10T03:18:13,113 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; OpenRegionProcedure 206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054}] 2024-11-10T03:18:13,269 INFO [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:13,269 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(7752): Opening region: {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} 2024-11-10T03:18:13,270 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 206af57796ad2b3d51883827ba0564a7 2024-11-10T03:18:13,270 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T03:18:13,270 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(7794): checking encryption for 206af57796ad2b3d51883827ba0564a7 2024-11-10T03:18:13,270 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(7797): checking classloading for 206af57796ad2b3d51883827ba0564a7 2024-11-10T03:18:13,272 INFO [StoreOpener-206af57796ad2b3d51883827ba0564a7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 206af57796ad2b3d51883827ba0564a7 2024-11-10T03:18:13,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-11-10T03:18:13,273 INFO [StoreOpener-206af57796ad2b3d51883827ba0564a7-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T03:18:13,273 INFO [StoreOpener-206af57796ad2b3d51883827ba0564a7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 206af57796ad2b3d51883827ba0564a7 columnFamilyName A 2024-11-10T03:18:13,273 DEBUG [StoreOpener-206af57796ad2b3d51883827ba0564a7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:18:13,274 INFO [StoreOpener-206af57796ad2b3d51883827ba0564a7-1 {}] regionserver.HStore(327): Store=206af57796ad2b3d51883827ba0564a7/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T03:18:13,274 INFO [StoreOpener-206af57796ad2b3d51883827ba0564a7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 206af57796ad2b3d51883827ba0564a7 2024-11-10T03:18:13,275 INFO [StoreOpener-206af57796ad2b3d51883827ba0564a7-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T03:18:13,275 INFO [StoreOpener-206af57796ad2b3d51883827ba0564a7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 206af57796ad2b3d51883827ba0564a7 columnFamilyName B 2024-11-10T03:18:13,276 DEBUG [StoreOpener-206af57796ad2b3d51883827ba0564a7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:18:13,276 INFO [StoreOpener-206af57796ad2b3d51883827ba0564a7-1 {}] regionserver.HStore(327): Store=206af57796ad2b3d51883827ba0564a7/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T03:18:13,276 INFO [StoreOpener-206af57796ad2b3d51883827ba0564a7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 206af57796ad2b3d51883827ba0564a7 2024-11-10T03:18:13,278 INFO [StoreOpener-206af57796ad2b3d51883827ba0564a7-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T03:18:13,278 INFO [StoreOpener-206af57796ad2b3d51883827ba0564a7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 206af57796ad2b3d51883827ba0564a7 columnFamilyName C 2024-11-10T03:18:13,278 DEBUG [StoreOpener-206af57796ad2b3d51883827ba0564a7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:18:13,278 INFO [StoreOpener-206af57796ad2b3d51883827ba0564a7-1 {}] regionserver.HStore(327): Store=206af57796ad2b3d51883827ba0564a7/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T03:18:13,278 INFO [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:13,279 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1038): replaying wal for 206af57796ad2b3d51883827ba0564a7 2024-11-10T03:18:13,279 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7 2024-11-10T03:18:13,279 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7 2024-11-10T03:18:13,280 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1048): stopping wal replay for 206af57796ad2b3d51883827ba0564a7 2024-11-10T03:18:13,280 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1060): Cleaning up temporary data for 206af57796ad2b3d51883827ba0564a7 2024-11-10T03:18:13,280 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T03:18:13,281 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1093): writing seq id for 206af57796ad2b3d51883827ba0564a7 2024-11-10T03:18:13,283 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T03:18:13,283 INFO [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1114): Opened 206af57796ad2b3d51883827ba0564a7; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67671716, jitterRate=0.008387148380279541}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T03:18:13,283 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 206af57796ad2b3d51883827ba0564a7 2024-11-10T03:18:13,283 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1006): Region open journal for 206af57796ad2b3d51883827ba0564a7: Running coprocessor pre-open hook at 1731208693270Writing region info on filesystem at 1731208693270Initializing all the Stores at 1731208693271 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731208693271Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731208693271Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731208693271Cleaning up temporary data from old regions at 1731208693280 (+9 ms)Running coprocessor post-open hooks at 1731208693283 (+3 ms)Region opened successfully at 1731208693283 2024-11-10T03:18:13,284 INFO [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., pid=116, masterSystemTime=1731208693265 2024-11-10T03:18:13,286 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:13,286 INFO [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:13,286 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=115 updating hbase:meta row=206af57796ad2b3d51883827ba0564a7, regionState=OPEN, openSeqNum=2, regionLocation=394d2222f724,38139,1731208572054 2024-11-10T03:18:13,288 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=116, ppid=115, state=RUNNABLE, hasLock=false; OpenRegionProcedure 206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 because future has completed 2024-11-10T03:18:13,290 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-10T03:18:13,290 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; OpenRegionProcedure 206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 in 175 msec 2024-11-10T03:18:13,292 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=115, resume processing ppid=114 2024-11-10T03:18:13,292 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, ppid=114, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=206af57796ad2b3d51883827ba0564a7, ASSIGN in 333 msec 2024-11-10T03:18:13,293 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-10T03:18:13,293 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731208693293"}]},"ts":"1731208693293"} 2024-11-10T03:18:13,295 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-10T03:18:13,296 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-10T03:18:13,297 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1640 sec 2024-11-10T03:18:14,271 WARN [regionserver/394d2222f724:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 6, running: 1 2024-11-10T03:18:14,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-11-10T03:18:14,283 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-10T03:18:14,284 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a027c2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:18:14,284 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 394d2222f724,37299,-1 for getting cluster id 2024-11-10T03:18:14,284 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T03:18:14,285 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f5c6727-65f7-4659-afe8-8e9a55664c78' 2024-11-10T03:18:14,285 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T03:18:14,285 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f5c6727-65f7-4659-afe8-8e9a55664c78" 2024-11-10T03:18:14,286 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2631cb07, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:18:14,286 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [394d2222f724,37299,-1] 2024-11-10T03:18:14,286 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T03:18:14,286 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:18:14,287 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42426, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T03:18:14,287 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5fcb6057, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:18:14,288 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T03:18:14,288 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=394d2222f724,38139,1731208572054, seqNum=-1] 2024-11-10T03:18:14,288 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T03:18:14,289 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46528, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T03:18:14,292 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-10T03:18:14,293 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 394d2222f724,37299,1731208571229 2024-11-10T03:18:14,293 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@76abb731 2024-11-10T03:18:14,293 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-10T03:18:14,294 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42432, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-10T03:18:14,296 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@243cd5d8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:18:14,296 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 394d2222f724,37299,-1 for getting cluster id 2024-11-10T03:18:14,296 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T03:18:14,297 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f5c6727-65f7-4659-afe8-8e9a55664c78' 2024-11-10T03:18:14,297 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T03:18:14,297 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f5c6727-65f7-4659-afe8-8e9a55664c78" 2024-11-10T03:18:14,297 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@cdac938, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:18:14,297 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [394d2222f724,37299,-1] 2024-11-10T03:18:14,298 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T03:18:14,298 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:18:14,298 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42450, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T03:18:14,301 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10812241, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:18:14,302 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10cc4573, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:18:14,302 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 394d2222f724,37299,-1 for getting cluster id 2024-11-10T03:18:14,302 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T03:18:14,302 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f5c6727-65f7-4659-afe8-8e9a55664c78' 2024-11-10T03:18:14,303 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T03:18:14,303 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f5c6727-65f7-4659-afe8-8e9a55664c78" 2024-11-10T03:18:14,303 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28614dd4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:18:14,303 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [394d2222f724,37299,-1] 2024-11-10T03:18:14,303 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T03:18:14,303 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:18:14,304 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42472, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T03:18:14,304 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bf3aded, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:18:14,305 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c9cf8ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:18:14,305 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 394d2222f724,37299,-1 for getting cluster id 2024-11-10T03:18:14,305 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T03:18:14,306 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f5c6727-65f7-4659-afe8-8e9a55664c78' 2024-11-10T03:18:14,306 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T03:18:14,306 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f5c6727-65f7-4659-afe8-8e9a55664c78" 2024-11-10T03:18:14,306 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@690a2990, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:18:14,306 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [394d2222f724,37299,-1] 2024-11-10T03:18:14,306 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T03:18:14,307 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:18:14,307 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42490, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T03:18:14,308 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@562154ce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:18:14,316 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19645622, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:18:14,316 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 394d2222f724,37299,-1 for getting cluster id 2024-11-10T03:18:14,316 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T03:18:14,317 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f5c6727-65f7-4659-afe8-8e9a55664c78' 2024-11-10T03:18:14,317 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T03:18:14,318 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f5c6727-65f7-4659-afe8-8e9a55664c78" 2024-11-10T03:18:14,318 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@403ec474, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:18:14,318 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [394d2222f724,37299,-1] 2024-11-10T03:18:14,318 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T03:18:14,318 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:18:14,319 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42506, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T03:18:14,319 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2083a466, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:18:14,320 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@364f2178, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:18:14,320 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 394d2222f724,37299,-1 for getting cluster id 2024-11-10T03:18:14,320 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T03:18:14,321 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f5c6727-65f7-4659-afe8-8e9a55664c78' 2024-11-10T03:18:14,321 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T03:18:14,321 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f5c6727-65f7-4659-afe8-8e9a55664c78" 2024-11-10T03:18:14,321 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29549358, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:18:14,321 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [394d2222f724,37299,-1] 2024-11-10T03:18:14,321 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T03:18:14,321 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:18:14,322 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42530, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T03:18:14,322 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63bebfd3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:18:14,323 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65b424a8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:18:14,323 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 394d2222f724,37299,-1 for getting cluster id 2024-11-10T03:18:14,323 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T03:18:14,324 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f5c6727-65f7-4659-afe8-8e9a55664c78' 2024-11-10T03:18:14,324 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T03:18:14,324 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f5c6727-65f7-4659-afe8-8e9a55664c78" 2024-11-10T03:18:14,324 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f75eead, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:18:14,324 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [394d2222f724,37299,-1] 2024-11-10T03:18:14,324 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T03:18:14,324 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:18:14,325 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42556, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T03:18:14,325 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56a4c4b5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:18:14,326 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c0a0b28, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:18:14,326 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 394d2222f724,37299,-1 for getting cluster id 2024-11-10T03:18:14,326 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T03:18:14,327 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f5c6727-65f7-4659-afe8-8e9a55664c78' 2024-11-10T03:18:14,327 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T03:18:14,327 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f5c6727-65f7-4659-afe8-8e9a55664c78" 2024-11-10T03:18:14,327 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@542d2d5e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:18:14,328 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [394d2222f724,37299,-1] 2024-11-10T03:18:14,328 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T03:18:14,328 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:18:14,329 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42572, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T03:18:14,329 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14dcfcf0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:18:14,330 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19b3abaf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:18:14,330 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 394d2222f724,37299,-1 for getting cluster id 2024-11-10T03:18:14,330 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T03:18:14,331 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f5c6727-65f7-4659-afe8-8e9a55664c78' 2024-11-10T03:18:14,331 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T03:18:14,331 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f5c6727-65f7-4659-afe8-8e9a55664c78" 2024-11-10T03:18:14,331 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@249b576c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:18:14,332 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [394d2222f724,37299,-1] 2024-11-10T03:18:14,332 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T03:18:14,332 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:18:14,332 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42596, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T03:18:14,333 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28d91341, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:18:14,333 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52bdc69, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:18:14,333 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 394d2222f724,37299,-1 for getting cluster id 2024-11-10T03:18:14,334 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T03:18:14,334 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f5c6727-65f7-4659-afe8-8e9a55664c78' 2024-11-10T03:18:14,334 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T03:18:14,334 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f5c6727-65f7-4659-afe8-8e9a55664c78" 2024-11-10T03:18:14,334 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@795c6553, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:18:14,334 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [394d2222f724,37299,-1] 2024-11-10T03:18:14,335 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T03:18:14,335 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:18:14,335 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42608, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T03:18:14,336 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f16a94e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:18:14,336 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7974c529, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:18:14,336 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 394d2222f724,37299,-1 for getting cluster id 2024-11-10T03:18:14,337 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T03:18:14,337 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f5c6727-65f7-4659-afe8-8e9a55664c78' 2024-11-10T03:18:14,337 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T03:18:14,337 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f5c6727-65f7-4659-afe8-8e9a55664c78" 2024-11-10T03:18:14,337 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ef763f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:18:14,337 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [394d2222f724,37299,-1] 2024-11-10T03:18:14,338 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T03:18:14,338 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:18:14,338 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42628, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T03:18:14,339 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75b6c28e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:18:14,340 DEBUG [Thread-2051 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T03:18:14,340 DEBUG [Thread-2040 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T03:18:14,340 DEBUG [Thread-2055 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T03:18:14,340 DEBUG [Thread-2042 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T03:18:14,340 DEBUG [Thread-2046 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T03:18:14,340 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=394d2222f724,38139,1731208572054, seqNum=-1] 2024-11-10T03:18:14,340 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=394d2222f724,38139,1731208572054, seqNum=-1] 2024-11-10T03:18:14,341 DEBUG [Thread-2044 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T03:18:14,341 DEBUG [Thread-2051 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T03:18:14,341 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T03:18:14,341 DEBUG [Thread-2040 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=394d2222f724,38139,1731208572054, seqNum=-1] 2024-11-10T03:18:14,341 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=394d2222f724,38139,1731208572054, seqNum=-1] 2024-11-10T03:18:14,341 DEBUG [Thread-2049 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T03:18:14,341 DEBUG [Thread-2040 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T03:18:14,341 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T03:18:14,341 DEBUG [Thread-2057 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T03:18:14,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=394d2222f724,38139,1731208572054, seqNum=-1] 2024-11-10T03:18:14,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T03:18:14,342 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46538, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T03:18:14,342 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46544, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T03:18:14,342 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=394d2222f724,38139,1731208572054, seqNum=-1] 2024-11-10T03:18:14,342 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=394d2222f724,38139,1731208572054, seqNum=-1] 2024-11-10T03:18:14,343 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T03:18:14,343 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=394d2222f724,38139,1731208572054, seqNum=-1] 2024-11-10T03:18:14,343 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T03:18:14,343 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T03:18:14,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2] 2024-11-10T03:18:14,344 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46548, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T03:18:14,344 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46562, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T03:18:14,344 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46584, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T03:18:14,344 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46592, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T03:18:14,344 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46568, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T03:18:14,345 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46600, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T03:18:14,346 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2] 2024-11-10T03:18:14,346 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2] 2024-11-10T03:18:14,347 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2] 2024-11-10T03:18:14,347 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2] 2024-11-10T03:18:14,347 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2] 2024-11-10T03:18:14,348 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2] 2024-11-10T03:18:14,348 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2] 2024-11-10T03:18:14,348 DEBUG [Thread-2038 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T03:18:14,349 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=394d2222f724,38139,1731208572054, seqNum=-1] 2024-11-10T03:18:14,349 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T03:18:14,351 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46608, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T03:18:14,352 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T03:18:14,352 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2] 2024-11-10T03:18:14,353 DEBUG [Thread-2053 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T03:18:14,353 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=394d2222f724,38139,1731208572054, seqNum=-1] 2024-11-10T03:18:14,354 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T03:18:14,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on 206af57796ad2b3d51883827ba0564a7 2024-11-10T03:18:14,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-11-10T03:18:14,355 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 206af57796ad2b3d51883827ba0564a7 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-10T03:18:14,357 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46612, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T03:18:14,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-10T03:18:14,358 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T03:18:14,358 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2] 2024-11-10T03:18:14,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 206af57796ad2b3d51883827ba0564a7, store=A 2024-11-10T03:18:14,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:18:14,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 206af57796ad2b3d51883827ba0564a7, store=B 2024-11-10T03:18:14,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:18:14,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 206af57796ad2b3d51883827ba0564a7, store=C 2024-11-10T03:18:14,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:18:14,360 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T03:18:14,360 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T03:18:14,384 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/A/acaf286142e44661a6c8898a5be63bba is 50, key is test_row_0/A:col10/1731208694355/Put/seqid=0 2024-11-10T03:18:14,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742281_1457 (size=12001) 2024-11-10T03:18:14,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:14,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46600 deadline: 1731208754388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:14,400 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:14,400 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:14,400 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:14,401 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731208694385, max timestamp=1731208694398Num uniques 859; 2024-11-10T03:18:14,401 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T03:18:14,401 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731208694385, max timestamp=1731208694398Num uniques 859; 2024-11-10T03:18:14,401 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T03:18:14,402 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731208694385, max timestamp=1731208694398Num uniques 859; 2024-11-10T03:18:14,402 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T03:18:14,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:14,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46608 deadline: 1731208754432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:14,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:14,444 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:14,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46548 deadline: 1731208754433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:14,444 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:14,444 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:14,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:14,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:14,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:14,445 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:14,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46544 deadline: 1731208754434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:14,445 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:14,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:14,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46562 deadline: 1731208754444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:14,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:14,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:14,446 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:14,446 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:14,446 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:14,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-10T03:18:14,510 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:14,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46600 deadline: 1731208754503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:14,511 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:14,511 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:14,511 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:14,516 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-10T03:18:14,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:14,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. as already flushing 2024-11-10T03:18:14,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:14,516 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:14,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:14,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:14,565 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:14,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46608 deadline: 1731208754553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:14,566 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:14,566 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:14,566 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:14,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46548 deadline: 1731208754553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:14,566 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:14,566 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:14,566 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:14,566 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:14,566 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:14,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46544 deadline: 1731208754553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:14,567 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:14,567 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:14,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:14,567 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:14,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46562 deadline: 1731208754554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:14,567 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:14,567 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:14,567 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:14,668 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-10T03:18:14,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:14,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. as already flushing 2024-11-10T03:18:14,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:14,669 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:14,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:14,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:14,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-10T03:18:14,731 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:14,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46600 deadline: 1731208754723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:14,732 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:14,732 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:14,732 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:14,775 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:14,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46544 deadline: 1731208754773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:14,776 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:14,776 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:14,776 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:14,776 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:14,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46608 deadline: 1731208754773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:14,776 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:14,776 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:14,776 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:14,776 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:14,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46562 deadline: 1731208754773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:14,777 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:14,777 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:14,777 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:14,777 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:14,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46548 deadline: 1731208754773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:14,777 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:14,777 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:14,777 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:14,791 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/A/acaf286142e44661a6c8898a5be63bba 2024-11-10T03:18:14,823 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-10T03:18:14,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:14,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. as already flushing 2024-11-10T03:18:14,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:14,824 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:14,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:14,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:14,844 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/B/b7a3861772e04ab29fe26f7c19438a20 is 50, key is test_row_0/B:col10/1731208694355/Put/seqid=0 2024-11-10T03:18:14,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742282_1458 (size=12001) 2024-11-10T03:18:14,859 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/B/b7a3861772e04ab29fe26f7c19438a20 2024-11-10T03:18:14,888 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/C/3a9c93cb787746b38064f12b2cee6994 is 50, key is test_row_0/C:col10/1731208694355/Put/seqid=0 2024-11-10T03:18:14,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742283_1459 (size=12001) 2024-11-10T03:18:14,918 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/C/3a9c93cb787746b38064f12b2cee6994 2024-11-10T03:18:14,922 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/A/acaf286142e44661a6c8898a5be63bba as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/acaf286142e44661a6c8898a5be63bba 2024-11-10T03:18:14,926 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/acaf286142e44661a6c8898a5be63bba, entries=150, sequenceid=12, filesize=11.7 K 2024-11-10T03:18:14,928 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/B/b7a3861772e04ab29fe26f7c19438a20 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/b7a3861772e04ab29fe26f7c19438a20 2024-11-10T03:18:14,937 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/b7a3861772e04ab29fe26f7c19438a20, entries=150, sequenceid=12, filesize=11.7 K 2024-11-10T03:18:14,942 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/C/3a9c93cb787746b38064f12b2cee6994 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/3a9c93cb787746b38064f12b2cee6994 2024-11-10T03:18:14,949 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/3a9c93cb787746b38064f12b2cee6994, entries=150, sequenceid=12, filesize=11.7 K 2024-11-10T03:18:14,950 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 206af57796ad2b3d51883827ba0564a7 in 596ms, sequenceid=12, compaction requested=false 2024-11-10T03:18:14,950 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 206af57796ad2b3d51883827ba0564a7: 2024-11-10T03:18:14,982 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-10T03:18:14,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:14,982 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing 206af57796ad2b3d51883827ba0564a7 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-11-10T03:18:14,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 206af57796ad2b3d51883827ba0564a7, store=A 2024-11-10T03:18:14,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T03:18:14,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 206af57796ad2b3d51883827ba0564a7, store=B 2024-11-10T03:18:14,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T03:18:14,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 206af57796ad2b3d51883827ba0564a7, store=C 2024-11-10T03:18:14,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T03:18:14,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-10T03:18:15,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/A/5c80644bf9e34688bfd6e319a7c12aa2 is 50, key is test_row_0/A:col10/1731208694442/Put/seqid=0 2024-11-10T03:18:15,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742284_1460 (size=12001) 2024-11-10T03:18:15,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on 206af57796ad2b3d51883827ba0564a7 2024-11-10T03:18:15,049 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. as already flushing 2024-11-10T03:18:15,094 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:15,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46548 deadline: 1731208755084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:15,095 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:15,095 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:15,095 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:15,095 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:15,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46544 deadline: 1731208755089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:15,095 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:15,095 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:15,095 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:15,095 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:15,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46600 deadline: 1731208755090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:15,096 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:15,096 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:15,096 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:15,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:15,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46608 deadline: 1731208755094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:15,099 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:15,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:15,099 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:15,099 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:15,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46562 deadline: 1731208755095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:15,099 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:15,099 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:15,099 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:15,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:15,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46600 deadline: 1731208755203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:15,209 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:15,209 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:15,209 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:15,212 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:15,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46608 deadline: 1731208755203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:15,212 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:15,212 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:15,212 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:15,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46562 deadline: 1731208755203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:15,212 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:15,213 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:15,213 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:15,213 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:15,419 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:15,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46600 deadline: 1731208755413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:15,419 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:15,419 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:15,419 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:15,422 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/A/5c80644bf9e34688bfd6e319a7c12aa2 2024-11-10T03:18:15,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/B/c6af7227b16448f38b1a2a17d026eafa is 50, key is test_row_0/B:col10/1731208694442/Put/seqid=0 2024-11-10T03:18:15,435 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:15,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46562 deadline: 1731208755423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:15,435 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:15,435 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:15,435 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:15,435 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:15,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46608 deadline: 1731208755423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:15,436 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:15,436 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:15,436 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:15,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742285_1461 (size=12001) 2024-11-10T03:18:15,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-10T03:18:15,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:15,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46548 deadline: 1731208755603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:15,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:15,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:15,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:15,615 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:15,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46544 deadline: 1731208755603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:15,616 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:15,616 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:15,616 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:15,734 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:15,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46600 deadline: 1731208755723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:15,734 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:15,734 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:15,734 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:15,755 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:15,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46562 deadline: 1731208755742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:15,755 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:15,755 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:15,755 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:15,755 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:15,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46608 deadline: 1731208755743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:15,755 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:15,756 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:15,756 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:15,839 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/B/c6af7227b16448f38b1a2a17d026eafa 2024-11-10T03:18:15,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/C/fda4c38e535d4448a280e1c0fd2e809d is 50, key is test_row_0/C:col10/1731208694442/Put/seqid=0 2024-11-10T03:18:15,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742286_1462 (size=12001) 2024-11-10T03:18:16,252 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:16,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46600 deadline: 1731208756243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:16,252 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:16,253 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:16,253 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:16,270 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/C/fda4c38e535d4448a280e1c0fd2e809d 2024-11-10T03:18:16,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:16,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46562 deadline: 1731208756263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:16,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:16,271 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:16,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46608 deadline: 1731208756263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:16,271 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:16,271 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:16,272 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:16,272 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:16,272 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:16,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/A/5c80644bf9e34688bfd6e319a7c12aa2 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/5c80644bf9e34688bfd6e319a7c12aa2 2024-11-10T03:18:16,277 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/5c80644bf9e34688bfd6e319a7c12aa2, entries=150, sequenceid=48, filesize=11.7 K 2024-11-10T03:18:16,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/B/c6af7227b16448f38b1a2a17d026eafa as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/c6af7227b16448f38b1a2a17d026eafa 2024-11-10T03:18:16,292 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/c6af7227b16448f38b1a2a17d026eafa, entries=150, sequenceid=48, filesize=11.7 K 2024-11-10T03:18:16,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/C/fda4c38e535d4448a280e1c0fd2e809d as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/fda4c38e535d4448a280e1c0fd2e809d 2024-11-10T03:18:16,298 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/fda4c38e535d4448a280e1c0fd2e809d, entries=150, sequenceid=48, filesize=11.7 K 2024-11-10T03:18:16,299 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for 206af57796ad2b3d51883827ba0564a7 in 1317ms, sequenceid=48, compaction requested=false 2024-11-10T03:18:16,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for 206af57796ad2b3d51883827ba0564a7: 2024-11-10T03:18:16,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:16,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-11-10T03:18:16,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-11-10T03:18:16,306 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-10T03:18:16,307 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9440 sec 2024-11-10T03:18:16,308 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 1.9550 sec 2024-11-10T03:18:16,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-10T03:18:16,503 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T03:18:16,504 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T03:18:16,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-11-10T03:18:16,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-10T03:18:16,508 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T03:18:16,508 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T03:18:16,508 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T03:18:16,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-10T03:18:16,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on 206af57796ad2b3d51883827ba0564a7 2024-11-10T03:18:16,629 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 206af57796ad2b3d51883827ba0564a7 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-10T03:18:16,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 206af57796ad2b3d51883827ba0564a7, store=A 2024-11-10T03:18:16,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:18:16,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 206af57796ad2b3d51883827ba0564a7, store=B 2024-11-10T03:18:16,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:18:16,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 206af57796ad2b3d51883827ba0564a7, store=C 2024-11-10T03:18:16,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:18:16,633 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/A/6b5595352c3b45bcadcbeabe7b80c93e is 50, key is test_row_0/A:col10/1731208696628/Put/seqid=0 2024-11-10T03:18:16,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742287_1463 (size=14341) 2024-11-10T03:18:16,659 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-10T03:18:16,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:16,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. as already flushing 2024-11-10T03:18:16,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:16,660 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:16,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:16,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:16,702 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:16,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46544 deadline: 1731208756691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:16,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:16,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:16,702 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:16,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:16,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46548 deadline: 1731208756696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:16,703 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:16,703 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:16,703 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:16,812 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-10T03:18:16,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:16,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. as already flushing 2024-11-10T03:18:16,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:16,812 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:16,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:16,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:16,823 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:16,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46544 deadline: 1731208756813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:16,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-10T03:18:16,823 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:16,823 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:16,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46548 deadline: 1731208756813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:16,823 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:16,823 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:16,824 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:16,824 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:16,824 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:16,965 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-10T03:18:16,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:16,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. as already flushing 2024-11-10T03:18:16,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:16,965 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:16,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:16,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:17,037 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/A/6b5595352c3b45bcadcbeabe7b80c93e 2024-11-10T03:18:17,038 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:17,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46544 deadline: 1731208757033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:17,038 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:17,038 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:17,039 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:17,039 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:17,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46548 deadline: 1731208757033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:17,039 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:17,039 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:17,039 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:17,042 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/B/f7e331178a3b44579cb38bf46f65e95c is 50, key is test_row_0/B:col10/1731208696628/Put/seqid=0 2024-11-10T03:18:17,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742288_1464 (size=12001) 2024-11-10T03:18:17,118 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-10T03:18:17,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:17,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. as already flushing 2024-11-10T03:18:17,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:17,118 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:17,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:17,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:17,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-10T03:18:17,271 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-10T03:18:17,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:17,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. as already flushing 2024-11-10T03:18:17,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:17,271 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:17,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:17,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:17,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:17,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46600 deadline: 1731208757263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:17,276 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 394d2222f724,38139,1731208572054 for 'test_row_0' in 206af57796ad2b3d51883827ba0564a7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T03:18:17,276 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:17,276 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:17,276 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:17,284 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-10T03:18:17,291 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:17,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46562 deadline: 1731208757282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:17,292 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:17,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46608 deadline: 1731208757282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:17,292 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 394d2222f724,38139,1731208572054 for 'test_row_1' in 206af57796ad2b3d51883827ba0564a7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T03:18:17,292 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:17,292 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:17,292 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:17,292 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 394d2222f724,38139,1731208572054 for 'test_row_0' in 206af57796ad2b3d51883827ba0564a7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2198 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T03:18:17,292 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:17,292 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:17,292 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:17,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:17,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46544 deadline: 1731208757343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:17,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:17,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46548 deadline: 1731208757343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:17,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:17,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:17,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:17,350 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:17,350 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:17,350 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:17,423 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-10T03:18:17,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:17,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. as already flushing 2024-11-10T03:18:17,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:17,424 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:17,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:17,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:17,445 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/B/f7e331178a3b44579cb38bf46f65e95c 2024-11-10T03:18:17,451 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/C/9188e932759742d6bd067d05751fdecd is 50, key is test_row_0/C:col10/1731208696628/Put/seqid=0 2024-11-10T03:18:17,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742289_1465 (size=12001) 2024-11-10T03:18:17,576 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-10T03:18:17,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:17,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. as already flushing 2024-11-10T03:18:17,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:17,577 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:17,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:17,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:17,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-10T03:18:17,730 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-10T03:18:17,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:17,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. as already flushing 2024-11-10T03:18:17,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:17,731 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:17,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:17,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:17,855 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/C/9188e932759742d6bd067d05751fdecd 2024-11-10T03:18:17,858 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/A/6b5595352c3b45bcadcbeabe7b80c93e as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/6b5595352c3b45bcadcbeabe7b80c93e 2024-11-10T03:18:17,866 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:17,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46548 deadline: 1731208757853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:17,867 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:17,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46544 deadline: 1731208757863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:17,868 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:17,868 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:17,868 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:17,869 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:17,869 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:17,869 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:17,876 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/6b5595352c3b45bcadcbeabe7b80c93e, entries=200, sequenceid=60, filesize=14.0 K 2024-11-10T03:18:17,879 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/B/f7e331178a3b44579cb38bf46f65e95c as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/f7e331178a3b44579cb38bf46f65e95c 2024-11-10T03:18:17,883 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-10T03:18:17,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:17,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. as already flushing 2024-11-10T03:18:17,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:17,883 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:17,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:17,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:17,891 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/f7e331178a3b44579cb38bf46f65e95c, entries=150, sequenceid=60, filesize=11.7 K 2024-11-10T03:18:17,892 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/C/9188e932759742d6bd067d05751fdecd as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/9188e932759742d6bd067d05751fdecd 2024-11-10T03:18:17,902 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/9188e932759742d6bd067d05751fdecd, entries=150, sequenceid=60, filesize=11.7 K 2024-11-10T03:18:17,903 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 206af57796ad2b3d51883827ba0564a7 in 1274ms, sequenceid=60, compaction requested=true 2024-11-10T03:18:17,903 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 206af57796ad2b3d51883827ba0564a7: 2024-11-10T03:18:17,903 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 206af57796ad2b3d51883827ba0564a7:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T03:18:17,903 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:18:17,903 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T03:18:17,903 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T03:18:17,903 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 206af57796ad2b3d51883827ba0564a7:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T03:18:17,903 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:18:17,903 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 206af57796ad2b3d51883827ba0564a7:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T03:18:17,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T03:18:17,904 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T03:18:17,904 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T03:18:17,904 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1541): 206af57796ad2b3d51883827ba0564a7/B is initiating minor compaction (all files) 2024-11-10T03:18:17,904 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1541): 206af57796ad2b3d51883827ba0564a7/A is initiating minor compaction (all files) 2024-11-10T03:18:17,904 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 206af57796ad2b3d51883827ba0564a7/B in TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:17,904 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 206af57796ad2b3d51883827ba0564a7/A in TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:17,904 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/b7a3861772e04ab29fe26f7c19438a20, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/c6af7227b16448f38b1a2a17d026eafa, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/f7e331178a3b44579cb38bf46f65e95c] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp, totalSize=35.2 K 2024-11-10T03:18:17,904 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/acaf286142e44661a6c8898a5be63bba, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/5c80644bf9e34688bfd6e319a7c12aa2, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/6b5595352c3b45bcadcbeabe7b80c93e] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp, totalSize=37.4 K 2024-11-10T03:18:17,905 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting b7a3861772e04ab29fe26f7c19438a20, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731208694347 2024-11-10T03:18:17,905 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting acaf286142e44661a6c8898a5be63bba, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731208694347 2024-11-10T03:18:17,905 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting c6af7227b16448f38b1a2a17d026eafa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731208694429 2024-11-10T03:18:17,905 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c80644bf9e34688bfd6e319a7c12aa2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731208694429 2024-11-10T03:18:17,905 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting f7e331178a3b44579cb38bf46f65e95c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731208695079 2024-11-10T03:18:17,906 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b5595352c3b45bcadcbeabe7b80c93e, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731208695048 2024-11-10T03:18:17,926 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 206af57796ad2b3d51883827ba0564a7#A#compaction#384 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:18:17,927 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/A/b737e77fc4884dfbab73048fa2a913d7 is 50, key is test_row_0/A:col10/1731208696628/Put/seqid=0 2024-11-10T03:18:17,942 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 206af57796ad2b3d51883827ba0564a7#B#compaction#385 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:18:17,943 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/B/ca58c833c8594e7d952c592ed25a9b47 is 50, key is test_row_0/B:col10/1731208696628/Put/seqid=0 2024-11-10T03:18:17,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742290_1466 (size=12104) 2024-11-10T03:18:17,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742291_1467 (size=12104) 2024-11-10T03:18:18,036 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-10T03:18:18,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:18,036 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing 206af57796ad2b3d51883827ba0564a7 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-10T03:18:18,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 206af57796ad2b3d51883827ba0564a7, store=A 2024-11-10T03:18:18,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:18:18,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 206af57796ad2b3d51883827ba0564a7, store=B 2024-11-10T03:18:18,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:18:18,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 206af57796ad2b3d51883827ba0564a7, store=C 2024-11-10T03:18:18,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:18:18,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/A/ca1c182e3834440699310febb62baa93 is 50, key is test_row_0/A:col10/1731208696687/Put/seqid=0 2024-11-10T03:18:18,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742292_1468 (size=12001) 2024-11-10T03:18:18,361 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/A/b737e77fc4884dfbab73048fa2a913d7 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/b737e77fc4884dfbab73048fa2a913d7 2024-11-10T03:18:18,363 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 206af57796ad2b3d51883827ba0564a7/A of 206af57796ad2b3d51883827ba0564a7 into b737e77fc4884dfbab73048fa2a913d7(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:18:18,363 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 206af57796ad2b3d51883827ba0564a7: 2024-11-10T03:18:18,363 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., storeName=206af57796ad2b3d51883827ba0564a7/A, priority=13, startTime=1731208697903; duration=0sec 2024-11-10T03:18:18,363 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T03:18:18,363 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 206af57796ad2b3d51883827ba0564a7:A 2024-11-10T03:18:18,364 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T03:18:18,364 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T03:18:18,364 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1541): 206af57796ad2b3d51883827ba0564a7/C is initiating minor compaction (all files) 2024-11-10T03:18:18,364 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 206af57796ad2b3d51883827ba0564a7/C in TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:18,364 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/3a9c93cb787746b38064f12b2cee6994, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/fda4c38e535d4448a280e1c0fd2e809d, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/9188e932759742d6bd067d05751fdecd] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp, totalSize=35.2 K 2024-11-10T03:18:18,364 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3a9c93cb787746b38064f12b2cee6994, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731208694347 2024-11-10T03:18:18,365 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting fda4c38e535d4448a280e1c0fd2e809d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731208694429 2024-11-10T03:18:18,365 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9188e932759742d6bd067d05751fdecd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731208695079 2024-11-10T03:18:18,371 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 206af57796ad2b3d51883827ba0564a7#C#compaction#387 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:18:18,371 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/C/12effa3c9bd44c8c84c6d610c416c15a is 50, key is test_row_0/C:col10/1731208696628/Put/seqid=0 2024-11-10T03:18:18,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742293_1469 (size=12104) 2024-11-10T03:18:18,380 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/B/ca58c833c8594e7d952c592ed25a9b47 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/ca58c833c8594e7d952c592ed25a9b47 2024-11-10T03:18:18,383 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 206af57796ad2b3d51883827ba0564a7/B of 206af57796ad2b3d51883827ba0564a7 into ca58c833c8594e7d952c592ed25a9b47(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:18:18,383 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 206af57796ad2b3d51883827ba0564a7: 2024-11-10T03:18:18,383 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., storeName=206af57796ad2b3d51883827ba0564a7/B, priority=13, startTime=1731208697903; duration=0sec 2024-11-10T03:18:18,383 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:18:18,383 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 206af57796ad2b3d51883827ba0564a7:B 2024-11-10T03:18:18,448 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/A/ca1c182e3834440699310febb62baa93 2024-11-10T03:18:18,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/B/269c5d74a50e497ba998dd3ffa30ab06 is 50, key is test_row_0/B:col10/1731208696687/Put/seqid=0 2024-11-10T03:18:18,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742294_1470 (size=12001) 2024-11-10T03:18:18,457 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/B/269c5d74a50e497ba998dd3ffa30ab06 2024-11-10T03:18:18,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/C/58b14f331f7d467ba8d7cbf299129e71 is 50, key is test_row_0/C:col10/1731208696687/Put/seqid=0 2024-11-10T03:18:18,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742295_1471 (size=12001) 2024-11-10T03:18:18,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-10T03:18:18,778 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/C/12effa3c9bd44c8c84c6d610c416c15a as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/12effa3c9bd44c8c84c6d610c416c15a 2024-11-10T03:18:18,782 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 206af57796ad2b3d51883827ba0564a7/C of 206af57796ad2b3d51883827ba0564a7 into 12effa3c9bd44c8c84c6d610c416c15a(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:18:18,782 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 206af57796ad2b3d51883827ba0564a7: 2024-11-10T03:18:18,782 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., storeName=206af57796ad2b3d51883827ba0564a7/C, priority=13, startTime=1731208697903; duration=0sec 2024-11-10T03:18:18,783 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:18:18,783 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 206af57796ad2b3d51883827ba0564a7:C 2024-11-10T03:18:18,866 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/C/58b14f331f7d467ba8d7cbf299129e71 2024-11-10T03:18:18,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/A/ca1c182e3834440699310febb62baa93 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/ca1c182e3834440699310febb62baa93 2024-11-10T03:18:18,873 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/ca1c182e3834440699310febb62baa93, entries=150, sequenceid=80, filesize=11.7 K 2024-11-10T03:18:18,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/B/269c5d74a50e497ba998dd3ffa30ab06 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/269c5d74a50e497ba998dd3ffa30ab06 2024-11-10T03:18:18,875 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. as already flushing 2024-11-10T03:18:18,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on 206af57796ad2b3d51883827ba0564a7 2024-11-10T03:18:18,878 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/269c5d74a50e497ba998dd3ffa30ab06, entries=150, sequenceid=80, filesize=11.7 K 2024-11-10T03:18:18,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/C/58b14f331f7d467ba8d7cbf299129e71 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/58b14f331f7d467ba8d7cbf299129e71 2024-11-10T03:18:18,906 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/58b14f331f7d467ba8d7cbf299129e71, entries=150, sequenceid=80, filesize=11.7 K 2024-11-10T03:18:18,921 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=46.96 KB/48090 for 206af57796ad2b3d51883827ba0564a7 in 885ms, sequenceid=80, compaction requested=false 2024-11-10T03:18:18,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for 206af57796ad2b3d51883827ba0564a7: 2024-11-10T03:18:18,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on 206af57796ad2b3d51883827ba0564a7 2024-11-10T03:18:18,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:18,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-11-10T03:18:18,921 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 206af57796ad2b3d51883827ba0564a7 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-10T03:18:18,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-11-10T03:18:18,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 206af57796ad2b3d51883827ba0564a7, store=A 2024-11-10T03:18:18,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:18:18,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 206af57796ad2b3d51883827ba0564a7, store=B 2024-11-10T03:18:18,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:18:18,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 206af57796ad2b3d51883827ba0564a7, store=C 2024-11-10T03:18:18,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:18:18,926 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-10T03:18:18,926 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4150 sec 2024-11-10T03:18:18,927 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 2.4220 sec 2024-11-10T03:18:18,931 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/A/e3a5cde597df41acb2c0a7ad8c21474d is 50, key is test_row_0/A:col10/1731208698901/Put/seqid=0 2024-11-10T03:18:18,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742296_1472 (size=14341) 2024-11-10T03:18:19,010 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:19,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46544 deadline: 1731208759001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:19,010 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:19,010 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:19,010 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:19,012 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:19,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46548 deadline: 1731208759010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:19,012 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:19,012 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:19,013 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:19,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:19,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46544 deadline: 1731208759112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:19,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:19,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:19,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:19,130 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:19,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46548 deadline: 1731208759123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:19,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:19,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:19,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:19,311 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:19,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46600 deadline: 1731208759302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:19,312 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:19,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46562 deadline: 1731208759303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:19,312 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 394d2222f724,38139,1731208572054 for 'test_row_0' in 206af57796ad2b3d51883827ba0564a7 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4222 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T03:18:19,313 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:19,313 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:19,312 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 394d2222f724,38139,1731208572054 for 'test_row_1' in 206af57796ad2b3d51883827ba0564a7 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4218 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T03:18:19,313 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:19,313 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:19,313 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:19,313 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:19,318 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:19,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46608 deadline: 1731208759313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:19,319 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 394d2222f724,38139,1731208572054 for 'test_row_0' in 206af57796ad2b3d51883827ba0564a7 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4225 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T03:18:19,319 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:19,319 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:19,319 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:19,330 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:19,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46544 deadline: 1731208759323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:19,330 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:19,330 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:19,330 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:19,335 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/A/e3a5cde597df41acb2c0a7ad8c21474d 2024-11-10T03:18:19,340 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/B/86f143f9a349474ab7cd2fbff6e14a9d is 50, key is test_row_0/B:col10/1731208698901/Put/seqid=0 2024-11-10T03:18:19,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742297_1473 (size=12001) 2024-11-10T03:18:19,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:19,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46548 deadline: 1731208759344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:19,350 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:19,350 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:19,350 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:19,651 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:19,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46544 deadline: 1731208759643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:19,652 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:19,652 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:19,652 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:19,667 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:19,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46548 deadline: 1731208759663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:19,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:19,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:19,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:19,752 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/B/86f143f9a349474ab7cd2fbff6e14a9d 2024-11-10T03:18:19,761 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/C/fec6fe5cc52646b9943935d57cc89d31 is 50, key is test_row_0/C:col10/1731208698901/Put/seqid=0 2024-11-10T03:18:19,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742298_1474 (size=12001) 2024-11-10T03:18:20,166 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/C/fec6fe5cc52646b9943935d57cc89d31 2024-11-10T03:18:20,169 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/A/e3a5cde597df41acb2c0a7ad8c21474d as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/e3a5cde597df41acb2c0a7ad8c21474d 2024-11-10T03:18:20,174 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:20,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46544 deadline: 1731208760163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:20,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:20,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:20,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:20,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:20,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46548 deadline: 1731208760173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:20,182 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:20,183 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:20,183 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:20,185 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/e3a5cde597df41acb2c0a7ad8c21474d, entries=200, sequenceid=95, filesize=14.0 K 2024-11-10T03:18:20,185 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/B/86f143f9a349474ab7cd2fbff6e14a9d as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/86f143f9a349474ab7cd2fbff6e14a9d 2024-11-10T03:18:20,188 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/86f143f9a349474ab7cd2fbff6e14a9d, entries=150, sequenceid=95, filesize=11.7 K 2024-11-10T03:18:20,189 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/C/fec6fe5cc52646b9943935d57cc89d31 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/fec6fe5cc52646b9943935d57cc89d31 2024-11-10T03:18:20,194 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/fec6fe5cc52646b9943935d57cc89d31, entries=150, sequenceid=95, filesize=11.7 K 2024-11-10T03:18:20,195 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 206af57796ad2b3d51883827ba0564a7 in 1274ms, sequenceid=95, compaction requested=true 2024-11-10T03:18:20,195 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 206af57796ad2b3d51883827ba0564a7: 2024-11-10T03:18:20,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 206af57796ad2b3d51883827ba0564a7:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T03:18:20,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:18:20,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 206af57796ad2b3d51883827ba0564a7:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T03:18:20,195 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T03:18:20,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T03:18:20,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 206af57796ad2b3d51883827ba0564a7:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T03:18:20,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-10T03:18:20,196 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T03:18:20,198 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T03:18:20,198 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T03:18:20,198 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1541): 206af57796ad2b3d51883827ba0564a7/B is initiating minor compaction (all files) 2024-11-10T03:18:20,198 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1541): 206af57796ad2b3d51883827ba0564a7/A is initiating minor compaction (all files) 2024-11-10T03:18:20,198 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 206af57796ad2b3d51883827ba0564a7/B in TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:20,198 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 206af57796ad2b3d51883827ba0564a7/A in TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:20,198 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/b737e77fc4884dfbab73048fa2a913d7, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/ca1c182e3834440699310febb62baa93, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/e3a5cde597df41acb2c0a7ad8c21474d] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp, totalSize=37.5 K 2024-11-10T03:18:20,198 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/ca58c833c8594e7d952c592ed25a9b47, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/269c5d74a50e497ba998dd3ffa30ab06, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/86f143f9a349474ab7cd2fbff6e14a9d] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp, totalSize=35.3 K 2024-11-10T03:18:20,198 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting b737e77fc4884dfbab73048fa2a913d7, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731208695079 2024-11-10T03:18:20,198 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting ca58c833c8594e7d952c592ed25a9b47, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731208695079 2024-11-10T03:18:20,199 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting ca1c182e3834440699310febb62baa93, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1731208696687 2024-11-10T03:18:20,200 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting e3a5cde597df41acb2c0a7ad8c21474d, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731208698889 2024-11-10T03:18:20,200 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 269c5d74a50e497ba998dd3ffa30ab06, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1731208696687 2024-11-10T03:18:20,200 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 86f143f9a349474ab7cd2fbff6e14a9d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731208698889 2024-11-10T03:18:20,210 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 206af57796ad2b3d51883827ba0564a7#A#compaction#393 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:18:20,211 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/A/0a6dc3cda8664244be5f6e19869248af is 50, key is test_row_0/A:col10/1731208698901/Put/seqid=0 2024-11-10T03:18:20,213 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 206af57796ad2b3d51883827ba0564a7#B#compaction#394 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:18:20,214 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/B/3da789b3418b40f19c464b34ba1b10ee is 50, key is test_row_0/B:col10/1731208698901/Put/seqid=0 2024-11-10T03:18:20,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742299_1475 (size=12207) 2024-11-10T03:18:20,222 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/A/0a6dc3cda8664244be5f6e19869248af as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/0a6dc3cda8664244be5f6e19869248af 2024-11-10T03:18:20,226 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 206af57796ad2b3d51883827ba0564a7/A of 206af57796ad2b3d51883827ba0564a7 into 0a6dc3cda8664244be5f6e19869248af(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:18:20,226 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 206af57796ad2b3d51883827ba0564a7: 2024-11-10T03:18:20,226 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., storeName=206af57796ad2b3d51883827ba0564a7/A, priority=13, startTime=1731208700195; duration=0sec 2024-11-10T03:18:20,226 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T03:18:20,226 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 206af57796ad2b3d51883827ba0564a7:A 2024-11-10T03:18:20,226 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T03:18:20,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742300_1476 (size=12207) 2024-11-10T03:18:20,227 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T03:18:20,227 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1541): 206af57796ad2b3d51883827ba0564a7/C is initiating minor compaction (all files) 2024-11-10T03:18:20,227 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 206af57796ad2b3d51883827ba0564a7/C in TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:20,227 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/12effa3c9bd44c8c84c6d610c416c15a, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/58b14f331f7d467ba8d7cbf299129e71, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/fec6fe5cc52646b9943935d57cc89d31] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp, totalSize=35.3 K 2024-11-10T03:18:20,228 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 12effa3c9bd44c8c84c6d610c416c15a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731208695079 2024-11-10T03:18:20,228 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 58b14f331f7d467ba8d7cbf299129e71, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1731208696687 2024-11-10T03:18:20,228 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting fec6fe5cc52646b9943935d57cc89d31, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731208698889 2024-11-10T03:18:20,234 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 206af57796ad2b3d51883827ba0564a7#C#compaction#395 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:18:20,234 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/C/4c9c972d2bef45c39156fabc8e1e5ea9 is 50, key is test_row_0/C:col10/1731208698901/Put/seqid=0 2024-11-10T03:18:20,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742301_1477 (size=12207) 2024-11-10T03:18:20,630 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/B/3da789b3418b40f19c464b34ba1b10ee as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/3da789b3418b40f19c464b34ba1b10ee 2024-11-10T03:18:20,634 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 206af57796ad2b3d51883827ba0564a7/B of 206af57796ad2b3d51883827ba0564a7 into 3da789b3418b40f19c464b34ba1b10ee(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:18:20,634 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 206af57796ad2b3d51883827ba0564a7: 2024-11-10T03:18:20,634 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., storeName=206af57796ad2b3d51883827ba0564a7/B, priority=13, startTime=1731208700195; duration=0sec 2024-11-10T03:18:20,634 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:18:20,634 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 206af57796ad2b3d51883827ba0564a7:B 2024-11-10T03:18:20,646 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/C/4c9c972d2bef45c39156fabc8e1e5ea9 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/4c9c972d2bef45c39156fabc8e1e5ea9 2024-11-10T03:18:20,650 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 206af57796ad2b3d51883827ba0564a7/C of 206af57796ad2b3d51883827ba0564a7 into 4c9c972d2bef45c39156fabc8e1e5ea9(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:18:20,650 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 206af57796ad2b3d51883827ba0564a7: 2024-11-10T03:18:20,650 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., storeName=206af57796ad2b3d51883827ba0564a7/C, priority=13, startTime=1731208700195; duration=0sec 2024-11-10T03:18:20,650 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:18:20,650 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 206af57796ad2b3d51883827ba0564a7:C 2024-11-10T03:18:20,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-10T03:18:20,673 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T03:18:20,674 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T03:18:20,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-10T03:18:20,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-10T03:18:20,678 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T03:18:20,678 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T03:18:20,679 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T03:18:20,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-10T03:18:20,831 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-10T03:18:20,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:20,832 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing 206af57796ad2b3d51883827ba0564a7 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-10T03:18:20,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 206af57796ad2b3d51883827ba0564a7, store=A 2024-11-10T03:18:20,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:18:20,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 206af57796ad2b3d51883827ba0564a7, store=B 2024-11-10T03:18:20,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:18:20,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 206af57796ad2b3d51883827ba0564a7, store=C 2024-11-10T03:18:20,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:18:20,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/A/57e3d38f7be1476c9d28596bd2a5c0ca is 50, key is test_row_0/A:col10/1731208698999/Put/seqid=0 2024-11-10T03:18:20,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742302_1478 (size=12001) 2024-11-10T03:18:20,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-10T03:18:21,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on 206af57796ad2b3d51883827ba0564a7 2024-11-10T03:18:21,188 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. as already flushing 2024-11-10T03:18:21,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:21,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46548 deadline: 1731208761210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:21,214 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:21,215 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:21,215 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:21,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:21,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46544 deadline: 1731208761214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:21,219 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:21,219 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:21,219 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:21,240 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/A/57e3d38f7be1476c9d28596bd2a5c0ca 2024-11-10T03:18:21,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/B/320ff5ac0d7646148c237be62db37864 is 50, key is test_row_0/B:col10/1731208698999/Put/seqid=0 2024-11-10T03:18:21,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742303_1479 (size=12001) 2024-11-10T03:18:21,250 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/B/320ff5ac0d7646148c237be62db37864 2024-11-10T03:18:21,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/C/720435b6c54c423896ed222d8a9f91bf is 50, key is test_row_0/C:col10/1731208698999/Put/seqid=0 2024-11-10T03:18:21,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742304_1480 (size=12001) 2024-11-10T03:18:21,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-10T03:18:21,323 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:21,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46544 deadline: 1731208761323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:21,324 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:21,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:21,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:21,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46548 deadline: 1731208761323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:21,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:21,324 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:21,324 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:21,324 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:21,538 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:21,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46544 deadline: 1731208761533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:21,538 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:21,538 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:21,538 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:21,538 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:21,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46548 deadline: 1731208761534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:21,538 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:21,538 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:21,539 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:21,662 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/C/720435b6c54c423896ed222d8a9f91bf 2024-11-10T03:18:21,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/A/57e3d38f7be1476c9d28596bd2a5c0ca as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/57e3d38f7be1476c9d28596bd2a5c0ca 2024-11-10T03:18:21,674 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/57e3d38f7be1476c9d28596bd2a5c0ca, entries=150, sequenceid=118, filesize=11.7 K 2024-11-10T03:18:21,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/B/320ff5ac0d7646148c237be62db37864 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/320ff5ac0d7646148c237be62db37864 2024-11-10T03:18:21,679 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/320ff5ac0d7646148c237be62db37864, entries=150, sequenceid=118, filesize=11.7 K 2024-11-10T03:18:21,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/C/720435b6c54c423896ed222d8a9f91bf as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/720435b6c54c423896ed222d8a9f91bf 2024-11-10T03:18:21,683 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/720435b6c54c423896ed222d8a9f91bf, entries=150, sequenceid=118, filesize=11.7 K 2024-11-10T03:18:21,684 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 206af57796ad2b3d51883827ba0564a7 in 853ms, sequenceid=118, compaction requested=false 2024-11-10T03:18:21,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for 206af57796ad2b3d51883827ba0564a7: 2024-11-10T03:18:21,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:21,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-10T03:18:21,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-10T03:18:21,686 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-10T03:18:21,686 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0060 sec 2024-11-10T03:18:21,688 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 1.0130 sec 2024-11-10T03:18:21,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-10T03:18:21,813 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T03:18:21,813 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T03:18:21,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-10T03:18:21,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-10T03:18:21,815 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T03:18:21,815 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T03:18:21,816 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T03:18:21,846 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 206af57796ad2b3d51883827ba0564a7 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-10T03:18:21,846 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 206af57796ad2b3d51883827ba0564a7, store=A 2024-11-10T03:18:21,846 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:18:21,846 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 206af57796ad2b3d51883827ba0564a7, store=B 2024-11-10T03:18:21,846 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:18:21,846 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 206af57796ad2b3d51883827ba0564a7, store=C 2024-11-10T03:18:21,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:18:21,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on 206af57796ad2b3d51883827ba0564a7 2024-11-10T03:18:21,870 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/A/617729d416464d458a31e69855fd0071 is 50, key is test_row_0/A:col10/1731208701844/Put/seqid=0 2024-11-10T03:18:21,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742305_1481 (size=9757) 2024-11-10T03:18:21,902 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/A/617729d416464d458a31e69855fd0071 2024-11-10T03:18:21,909 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/B/18d4552c238544f383ea470a2031da86 is 50, key is test_row_0/B:col10/1731208701844/Put/seqid=0 2024-11-10T03:18:21,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742306_1482 (size=9757) 2024-11-10T03:18:21,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-10T03:18:21,924 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:21,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46544 deadline: 1731208761912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:21,924 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:21,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:21,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:21,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46548 deadline: 1731208761918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:21,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:21,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:21,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:21,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:21,967 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-10T03:18:21,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:21,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. as already flushing 2024-11-10T03:18:21,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:21,968 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:21,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:21,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:22,043 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:22,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46544 deadline: 1731208762033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:22,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:22,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:22,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:22,047 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:22,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46548 deadline: 1731208762041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:22,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:22,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:22,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:22,120 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-10T03:18:22,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:22,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. as already flushing 2024-11-10T03:18:22,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:22,120 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:22,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:22,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:22,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-10T03:18:22,256 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:22,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46544 deadline: 1731208762253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:22,257 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:22,257 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:22,257 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:22,257 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:22,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46548 deadline: 1731208762253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:22,257 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:22,257 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:22,257 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:22,273 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-10T03:18:22,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:22,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. as already flushing 2024-11-10T03:18:22,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:22,273 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:22,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:22,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:22,314 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/B/18d4552c238544f383ea470a2031da86 2024-11-10T03:18:22,318 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/C/1089d5af2d3f459c9341cf672a203046 is 50, key is test_row_0/C:col10/1731208701844/Put/seqid=0 2024-11-10T03:18:22,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742307_1483 (size=9757) 2024-11-10T03:18:22,426 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-10T03:18:22,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:22,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. as already flushing 2024-11-10T03:18:22,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:22,426 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:22,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:22,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:22,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-10T03:18:22,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:22,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46544 deadline: 1731208762563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:22,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:22,567 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:22,567 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:22,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46548 deadline: 1731208762563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:22,567 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:22,567 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:22,567 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:22,568 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:22,578 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-10T03:18:22,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:22,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. as already flushing 2024-11-10T03:18:22,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:22,579 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:22,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:22,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:22,727 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/C/1089d5af2d3f459c9341cf672a203046 2024-11-10T03:18:22,730 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/A/617729d416464d458a31e69855fd0071 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/617729d416464d458a31e69855fd0071 2024-11-10T03:18:22,731 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-10T03:18:22,733 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/617729d416464d458a31e69855fd0071, entries=100, sequenceid=131, filesize=9.5 K 2024-11-10T03:18:22,734 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/B/18d4552c238544f383ea470a2031da86 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/18d4552c238544f383ea470a2031da86 2024-11-10T03:18:22,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:22,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. as already flushing 2024-11-10T03:18:22,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:22,734 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:22,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:22,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:22,737 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/18d4552c238544f383ea470a2031da86, entries=100, sequenceid=131, filesize=9.5 K 2024-11-10T03:18:22,739 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/C/1089d5af2d3f459c9341cf672a203046 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/1089d5af2d3f459c9341cf672a203046 2024-11-10T03:18:22,744 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/1089d5af2d3f459c9341cf672a203046, entries=100, sequenceid=131, filesize=9.5 K 2024-11-10T03:18:22,745 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 206af57796ad2b3d51883827ba0564a7 in 899ms, sequenceid=131, compaction requested=true 2024-11-10T03:18:22,745 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 206af57796ad2b3d51883827ba0564a7: 2024-11-10T03:18:22,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 206af57796ad2b3d51883827ba0564a7:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T03:18:22,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:18:22,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 206af57796ad2b3d51883827ba0564a7:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T03:18:22,745 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T03:18:22,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:18:22,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 206af57796ad2b3d51883827ba0564a7:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T03:18:22,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T03:18:22,745 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T03:18:22,746 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T03:18:22,746 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1541): 206af57796ad2b3d51883827ba0564a7/B is initiating minor compaction (all files) 2024-11-10T03:18:22,746 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T03:18:22,746 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 206af57796ad2b3d51883827ba0564a7/B in TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:22,746 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1541): 206af57796ad2b3d51883827ba0564a7/A is initiating minor compaction (all files) 2024-11-10T03:18:22,746 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 206af57796ad2b3d51883827ba0564a7/A in TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:22,746 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/3da789b3418b40f19c464b34ba1b10ee, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/320ff5ac0d7646148c237be62db37864, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/18d4552c238544f383ea470a2031da86] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp, totalSize=33.2 K 2024-11-10T03:18:22,746 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/0a6dc3cda8664244be5f6e19869248af, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/57e3d38f7be1476c9d28596bd2a5c0ca, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/617729d416464d458a31e69855fd0071] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp, totalSize=33.2 K 2024-11-10T03:18:22,746 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 3da789b3418b40f19c464b34ba1b10ee, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731208698889 2024-11-10T03:18:22,746 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0a6dc3cda8664244be5f6e19869248af, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731208698889 2024-11-10T03:18:22,747 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 320ff5ac0d7646148c237be62db37864, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731208698959 2024-11-10T03:18:22,747 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 57e3d38f7be1476c9d28596bd2a5c0ca, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731208698959 2024-11-10T03:18:22,747 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 18d4552c238544f383ea470a2031da86, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731208701213 2024-11-10T03:18:22,747 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 617729d416464d458a31e69855fd0071, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731208701213 2024-11-10T03:18:22,751 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 206af57796ad2b3d51883827ba0564a7#A#compaction#402 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:18:22,752 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/A/37e0da99c40042f6b4339618acf551f7 is 50, key is test_row_0/A:col10/1731208701844/Put/seqid=0 2024-11-10T03:18:22,752 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 206af57796ad2b3d51883827ba0564a7#B#compaction#403 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:18:22,752 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/B/15798386f8f746039e32e36caf1414d4 is 50, key is test_row_0/B:col10/1731208701844/Put/seqid=0 2024-11-10T03:18:22,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742308_1484 (size=12409) 2024-11-10T03:18:22,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742309_1485 (size=12409) 2024-11-10T03:18:22,886 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-10T03:18:22,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:22,887 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing 206af57796ad2b3d51883827ba0564a7 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-10T03:18:22,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 206af57796ad2b3d51883827ba0564a7, store=A 2024-11-10T03:18:22,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:18:22,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 206af57796ad2b3d51883827ba0564a7, store=B 2024-11-10T03:18:22,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:18:22,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 206af57796ad2b3d51883827ba0564a7, store=C 2024-11-10T03:18:22,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:18:22,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/A/be333081c3dd47d0bdbc1d0ee19ffb9a is 50, key is test_row_0/A:col10/1731208701904/Put/seqid=0 2024-11-10T03:18:22,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742310_1486 (size=12151) 2024-11-10T03:18:22,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-10T03:18:23,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on 206af57796ad2b3d51883827ba0564a7 2024-11-10T03:18:23,081 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. as already flushing 2024-11-10T03:18:23,124 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:23,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46544 deadline: 1731208763112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:23,124 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:23,124 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:23,124 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:23,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46548 deadline: 1731208763113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:23,124 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:23,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:23,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:23,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:23,160 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/A/37e0da99c40042f6b4339618acf551f7 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/37e0da99c40042f6b4339618acf551f7 2024-11-10T03:18:23,167 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 206af57796ad2b3d51883827ba0564a7/A of 206af57796ad2b3d51883827ba0564a7 into 37e0da99c40042f6b4339618acf551f7(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:18:23,167 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 206af57796ad2b3d51883827ba0564a7: 2024-11-10T03:18:23,167 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., storeName=206af57796ad2b3d51883827ba0564a7/A, priority=13, startTime=1731208702745; duration=0sec 2024-11-10T03:18:23,167 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T03:18:23,167 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 206af57796ad2b3d51883827ba0564a7:A 2024-11-10T03:18:23,167 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T03:18:23,168 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T03:18:23,168 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1541): 206af57796ad2b3d51883827ba0564a7/C is initiating minor compaction (all files) 2024-11-10T03:18:23,168 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 206af57796ad2b3d51883827ba0564a7/C in TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:23,168 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/4c9c972d2bef45c39156fabc8e1e5ea9, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/720435b6c54c423896ed222d8a9f91bf, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/1089d5af2d3f459c9341cf672a203046] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp, totalSize=33.2 K 2024-11-10T03:18:23,168 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/B/15798386f8f746039e32e36caf1414d4 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/15798386f8f746039e32e36caf1414d4 2024-11-10T03:18:23,168 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4c9c972d2bef45c39156fabc8e1e5ea9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731208698889 2024-11-10T03:18:23,169 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 720435b6c54c423896ed222d8a9f91bf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731208698959 2024-11-10T03:18:23,169 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1089d5af2d3f459c9341cf672a203046, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731208701213 2024-11-10T03:18:23,172 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 206af57796ad2b3d51883827ba0564a7/B of 206af57796ad2b3d51883827ba0564a7 into 15798386f8f746039e32e36caf1414d4(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:18:23,172 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 206af57796ad2b3d51883827ba0564a7: 2024-11-10T03:18:23,172 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., storeName=206af57796ad2b3d51883827ba0564a7/B, priority=13, startTime=1731208702745; duration=0sec 2024-11-10T03:18:23,172 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:18:23,172 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 206af57796ad2b3d51883827ba0564a7:B 2024-11-10T03:18:23,175 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 206af57796ad2b3d51883827ba0564a7#C#compaction#405 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:18:23,175 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/C/fddc2bd590ec41e193a4223a9839983d is 50, key is test_row_0/C:col10/1731208701844/Put/seqid=0 2024-11-10T03:18:23,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742311_1487 (size=12409) 2024-11-10T03:18:23,241 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:23,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46544 deadline: 1731208763233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:23,241 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:23,241 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:23,241 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:23,241 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:23,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46548 deadline: 1731208763233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:23,242 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:23,242 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:23,242 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:23,295 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=150 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/A/be333081c3dd47d0bdbc1d0ee19ffb9a 2024-11-10T03:18:23,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/B/1ece73c143124171a7b58e642b1696ed is 50, key is test_row_0/B:col10/1731208701904/Put/seqid=0 2024-11-10T03:18:23,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742312_1488 (size=12151) 2024-11-10T03:18:23,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:23,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46600 deadline: 1731208763333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:23,340 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 394d2222f724,38139,1731208572054 for 'test_row_0' in 206af57796ad2b3d51883827ba0564a7 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8250 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T03:18:23,340 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:23,340 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:23,340 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:23,354 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:23,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46608 deadline: 1731208763343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:23,355 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 394d2222f724,38139,1731208572054 for 'test_row_0' in 206af57796ad2b3d51883827ba0564a7 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8261 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T03:18:23,355 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:23,355 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:23,355 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:23,359 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:23,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46562 deadline: 1731208763353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:23,359 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 394d2222f724,38139,1731208572054 for 'test_row_1' in 206af57796ad2b3d51883827ba0564a7 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8265 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T03:18:23,360 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:23,360 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:23,360 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:23,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:23,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46544 deadline: 1731208763453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:23,463 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:23,463 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:23,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46548 deadline: 1731208763453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:23,463 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:23,463 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:23,463 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:23,463 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:23,463 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:23,584 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/C/fddc2bd590ec41e193a4223a9839983d as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/fddc2bd590ec41e193a4223a9839983d 2024-11-10T03:18:23,588 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 206af57796ad2b3d51883827ba0564a7/C of 206af57796ad2b3d51883827ba0564a7 into fddc2bd590ec41e193a4223a9839983d(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:18:23,588 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 206af57796ad2b3d51883827ba0564a7: 2024-11-10T03:18:23,588 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., storeName=206af57796ad2b3d51883827ba0564a7/C, priority=13, startTime=1731208702745; duration=0sec 2024-11-10T03:18:23,588 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:18:23,588 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 206af57796ad2b3d51883827ba0564a7:C 2024-11-10T03:18:23,720 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=150 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/B/1ece73c143124171a7b58e642b1696ed 2024-11-10T03:18:23,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/C/d40e9bd982e14363be018ff5f060144c is 50, key is test_row_0/C:col10/1731208701904/Put/seqid=0 2024-11-10T03:18:23,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742313_1489 (size=12151) 2024-11-10T03:18:23,765 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=150 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/C/d40e9bd982e14363be018ff5f060144c 2024-11-10T03:18:23,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/A/be333081c3dd47d0bdbc1d0ee19ffb9a as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/be333081c3dd47d0bdbc1d0ee19ffb9a 2024-11-10T03:18:23,778 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:23,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46548 deadline: 1731208763773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:23,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:23,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:23,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:23,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:23,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46544 deadline: 1731208763774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:23,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:23,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:23,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:23,786 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/be333081c3dd47d0bdbc1d0ee19ffb9a, entries=150, sequenceid=150, filesize=11.9 K 2024-11-10T03:18:23,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/B/1ece73c143124171a7b58e642b1696ed as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/1ece73c143124171a7b58e642b1696ed 2024-11-10T03:18:23,796 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/1ece73c143124171a7b58e642b1696ed, entries=150, sequenceid=150, filesize=11.9 K 2024-11-10T03:18:23,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/C/d40e9bd982e14363be018ff5f060144c as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/d40e9bd982e14363be018ff5f060144c 2024-11-10T03:18:23,801 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/d40e9bd982e14363be018ff5f060144c, entries=150, sequenceid=150, filesize=11.9 K 2024-11-10T03:18:23,802 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 206af57796ad2b3d51883827ba0564a7 in 914ms, sequenceid=150, compaction requested=false 2024-11-10T03:18:23,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for 206af57796ad2b3d51883827ba0564a7: 2024-11-10T03:18:23,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:23,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-10T03:18:23,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-10T03:18:23,805 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-10T03:18:23,805 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9880 sec 2024-11-10T03:18:23,807 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 1.9920 sec 2024-11-10T03:18:23,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-10T03:18:23,963 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T03:18:23,963 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T03:18:23,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-10T03:18:23,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-10T03:18:23,965 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T03:18:23,966 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T03:18:23,966 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T03:18:24,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-10T03:18:24,118 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-10T03:18:24,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:24,119 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing 206af57796ad2b3d51883827ba0564a7 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-10T03:18:24,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 206af57796ad2b3d51883827ba0564a7, store=A 2024-11-10T03:18:24,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:18:24,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 206af57796ad2b3d51883827ba0564a7, store=B 2024-11-10T03:18:24,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:18:24,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 206af57796ad2b3d51883827ba0564a7, store=C 2024-11-10T03:18:24,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:18:24,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/A/e7a0d9198a6a4b4fb39a71973f4444d2 is 50, key is test_row_0/A:col10/1731208703110/Put/seqid=0 2024-11-10T03:18:24,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742314_1490 (size=12151) 2024-11-10T03:18:24,153 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/A/e7a0d9198a6a4b4fb39a71973f4444d2 2024-11-10T03:18:24,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/B/259b42fcb4e545ff9feb9ded907019f5 is 50, key is test_row_0/B:col10/1731208703110/Put/seqid=0 2024-11-10T03:18:24,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742315_1491 (size=12151) 2024-11-10T03:18:24,189 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/B/259b42fcb4e545ff9feb9ded907019f5 2024-11-10T03:18:24,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/C/bae683e3ccf045f78601495689f2d5a1 is 50, key is test_row_0/C:col10/1731208703110/Put/seqid=0 2024-11-10T03:18:24,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742316_1492 (size=12151) 2024-11-10T03:18:24,212 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/C/bae683e3ccf045f78601495689f2d5a1 2024-11-10T03:18:24,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/A/e7a0d9198a6a4b4fb39a71973f4444d2 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/e7a0d9198a6a4b4fb39a71973f4444d2 2024-11-10T03:18:24,230 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/e7a0d9198a6a4b4fb39a71973f4444d2, entries=150, sequenceid=166, filesize=11.9 K 2024-11-10T03:18:24,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/B/259b42fcb4e545ff9feb9ded907019f5 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/259b42fcb4e545ff9feb9ded907019f5 2024-11-10T03:18:24,236 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/259b42fcb4e545ff9feb9ded907019f5, entries=150, sequenceid=166, filesize=11.9 K 2024-11-10T03:18:24,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/C/bae683e3ccf045f78601495689f2d5a1 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/bae683e3ccf045f78601495689f2d5a1 2024-11-10T03:18:24,240 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/bae683e3ccf045f78601495689f2d5a1, entries=150, sequenceid=166, filesize=11.9 K 2024-11-10T03:18:24,243 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=0 B/0 for 206af57796ad2b3d51883827ba0564a7 in 124ms, sequenceid=166, compaction requested=true 2024-11-10T03:18:24,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for 206af57796ad2b3d51883827ba0564a7: 2024-11-10T03:18:24,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:24,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-10T03:18:24,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-10T03:18:24,246 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-10T03:18:24,247 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 279 msec 2024-11-10T03:18:24,248 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 284 msec 2024-11-10T03:18:24,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-10T03:18:24,283 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T03:18:24,284 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T03:18:24,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-10T03:18:24,286 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T03:18:24,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-10T03:18:24,286 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T03:18:24,286 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T03:18:24,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on 206af57796ad2b3d51883827ba0564a7 2024-11-10T03:18:24,308 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 206af57796ad2b3d51883827ba0564a7 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-10T03:18:24,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 206af57796ad2b3d51883827ba0564a7, store=A 2024-11-10T03:18:24,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:18:24,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 206af57796ad2b3d51883827ba0564a7, store=B 2024-11-10T03:18:24,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:18:24,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 206af57796ad2b3d51883827ba0564a7, store=C 2024-11-10T03:18:24,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:18:24,311 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/A/27d4d992f2394073bb5067300066221b is 50, key is test_row_0/A:col10/1731208704301/Put/seqid=0 2024-11-10T03:18:24,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742317_1493 (size=14541) 2024-11-10T03:18:24,315 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/A/27d4d992f2394073bb5067300066221b 2024-11-10T03:18:24,320 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/B/6090c04d37554d87a8d998e3d5eb8f3a is 50, key is test_row_0/B:col10/1731208704301/Put/seqid=0 2024-11-10T03:18:24,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742318_1494 (size=12151) 2024-11-10T03:18:24,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-10T03:18:24,428 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731208704397, max timestamp=1731208704426Num uniques 859; 2024-11-10T03:18:24,428 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T03:18:24,428 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731208704397, max timestamp=1731208704426Num uniques 859; 2024-11-10T03:18:24,428 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T03:18:24,430 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731208704397, max timestamp=1731208704426Num uniques 859; 2024-11-10T03:18:24,430 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T03:18:24,438 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-10T03:18:24,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:24,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. as already flushing 2024-11-10T03:18:24,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:24,438 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:24,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:24,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:24,543 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:24,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46544 deadline: 1731208764534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:24,544 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:24,544 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:24,544 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:24,554 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:24,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46548 deadline: 1731208764541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:24,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:24,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:24,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:24,592 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-10T03:18:24,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:24,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. as already flushing 2024-11-10T03:18:24,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:24,596 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:24,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:24,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:24,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-10T03:18:24,668 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:24,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46544 deadline: 1731208764652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:24,668 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:24,668 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:24,668 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:24,668 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:24,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46548 deadline: 1731208764663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:24,669 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:24,669 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:24,669 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:24,730 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/B/6090c04d37554d87a8d998e3d5eb8f3a 2024-11-10T03:18:24,737 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/C/42c250f31b0b4c00a6232411adc09c4f is 50, key is test_row_0/C:col10/1731208704301/Put/seqid=0 2024-11-10T03:18:24,750 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-10T03:18:24,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:24,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. as already flushing 2024-11-10T03:18:24,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:24,750 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:24,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:24,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:24,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742319_1495 (size=12151) 2024-11-10T03:18:24,759 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/C/42c250f31b0b4c00a6232411adc09c4f 2024-11-10T03:18:24,763 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/A/27d4d992f2394073bb5067300066221b as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/27d4d992f2394073bb5067300066221b 2024-11-10T03:18:24,765 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/27d4d992f2394073bb5067300066221b, entries=200, sequenceid=176, filesize=14.2 K 2024-11-10T03:18:24,765 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/B/6090c04d37554d87a8d998e3d5eb8f3a as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/6090c04d37554d87a8d998e3d5eb8f3a 2024-11-10T03:18:24,767 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/6090c04d37554d87a8d998e3d5eb8f3a, entries=150, sequenceid=176, filesize=11.9 K 2024-11-10T03:18:24,768 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/C/42c250f31b0b4c00a6232411adc09c4f as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/42c250f31b0b4c00a6232411adc09c4f 2024-11-10T03:18:24,770 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/42c250f31b0b4c00a6232411adc09c4f, entries=150, sequenceid=176, filesize=11.9 K 2024-11-10T03:18:24,770 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 206af57796ad2b3d51883827ba0564a7 in 462ms, sequenceid=176, compaction requested=true 2024-11-10T03:18:24,770 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 206af57796ad2b3d51883827ba0564a7: 2024-11-10T03:18:24,770 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 206af57796ad2b3d51883827ba0564a7:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T03:18:24,770 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:18:24,770 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T03:18:24,770 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 206af57796ad2b3d51883827ba0564a7:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T03:18:24,770 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T03:18:24,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:18:24,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 206af57796ad2b3d51883827ba0564a7:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T03:18:24,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T03:18:24,771 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51252 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T03:18:24,771 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48862 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T03:18:24,771 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1541): 206af57796ad2b3d51883827ba0564a7/A is initiating minor compaction (all files) 2024-11-10T03:18:24,771 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1541): 206af57796ad2b3d51883827ba0564a7/B is initiating minor compaction (all files) 2024-11-10T03:18:24,771 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 206af57796ad2b3d51883827ba0564a7/A in TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:24,771 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 206af57796ad2b3d51883827ba0564a7/B in TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:24,771 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/37e0da99c40042f6b4339618acf551f7, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/be333081c3dd47d0bdbc1d0ee19ffb9a, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/e7a0d9198a6a4b4fb39a71973f4444d2, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/27d4d992f2394073bb5067300066221b] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp, totalSize=50.1 K 2024-11-10T03:18:24,771 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/15798386f8f746039e32e36caf1414d4, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/1ece73c143124171a7b58e642b1696ed, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/259b42fcb4e545ff9feb9ded907019f5, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/6090c04d37554d87a8d998e3d5eb8f3a] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp, totalSize=47.7 K 2024-11-10T03:18:24,772 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 37e0da99c40042f6b4339618acf551f7, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731208698959 2024-11-10T03:18:24,772 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 15798386f8f746039e32e36caf1414d4, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731208698959 2024-11-10T03:18:24,772 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting be333081c3dd47d0bdbc1d0ee19ffb9a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1731208701904 2024-11-10T03:18:24,772 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ece73c143124171a7b58e642b1696ed, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1731208701904 2024-11-10T03:18:24,773 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 259b42fcb4e545ff9feb9ded907019f5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1731208703104 2024-11-10T03:18:24,773 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting e7a0d9198a6a4b4fb39a71973f4444d2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1731208703104 2024-11-10T03:18:24,776 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 6090c04d37554d87a8d998e3d5eb8f3a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731208704301 2024-11-10T03:18:24,776 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 27d4d992f2394073bb5067300066221b, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731208704299 2024-11-10T03:18:24,792 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 206af57796ad2b3d51883827ba0564a7#B#compaction#414 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:18:24,793 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/B/9c241ab08a294a7da132353cab4c57fd is 50, key is test_row_0/B:col10/1731208704301/Put/seqid=0 2024-11-10T03:18:24,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742320_1496 (size=12595) 2024-11-10T03:18:24,800 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 206af57796ad2b3d51883827ba0564a7#A#compaction#415 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:18:24,800 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/A/5693f7b5f2ec4310ada7cc9529ec32c6 is 50, key is test_row_0/A:col10/1731208704301/Put/seqid=0 2024-11-10T03:18:24,802 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/B/9c241ab08a294a7da132353cab4c57fd as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/9c241ab08a294a7da132353cab4c57fd 2024-11-10T03:18:24,806 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 206af57796ad2b3d51883827ba0564a7/B of 206af57796ad2b3d51883827ba0564a7 into 9c241ab08a294a7da132353cab4c57fd(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:18:24,806 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 206af57796ad2b3d51883827ba0564a7: 2024-11-10T03:18:24,806 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., storeName=206af57796ad2b3d51883827ba0564a7/B, priority=12, startTime=1731208704770; duration=0sec 2024-11-10T03:18:24,806 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T03:18:24,807 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 206af57796ad2b3d51883827ba0564a7:B 2024-11-10T03:18:24,807 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T03:18:24,808 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48862 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T03:18:24,808 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1541): 206af57796ad2b3d51883827ba0564a7/C is initiating minor compaction (all files) 2024-11-10T03:18:24,808 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 206af57796ad2b3d51883827ba0564a7/C in TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:24,808 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/fddc2bd590ec41e193a4223a9839983d, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/d40e9bd982e14363be018ff5f060144c, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/bae683e3ccf045f78601495689f2d5a1, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/42c250f31b0b4c00a6232411adc09c4f] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp, totalSize=47.7 K 2024-11-10T03:18:24,808 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting fddc2bd590ec41e193a4223a9839983d, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731208698959 2024-11-10T03:18:24,808 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting d40e9bd982e14363be018ff5f060144c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1731208701904 2024-11-10T03:18:24,809 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting bae683e3ccf045f78601495689f2d5a1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1731208703104 2024-11-10T03:18:24,809 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 42c250f31b0b4c00a6232411adc09c4f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731208704301 2024-11-10T03:18:24,815 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 206af57796ad2b3d51883827ba0564a7#C#compaction#416 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:18:24,815 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/C/15213684846c4f2e9df138df025f81fb is 50, key is test_row_0/C:col10/1731208704301/Put/seqid=0 2024-11-10T03:18:24,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742321_1497 (size=12595) 2024-11-10T03:18:24,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742322_1498 (size=12595) 2024-11-10T03:18:24,832 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/C/15213684846c4f2e9df138df025f81fb as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/15213684846c4f2e9df138df025f81fb 2024-11-10T03:18:24,835 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 206af57796ad2b3d51883827ba0564a7/C of 206af57796ad2b3d51883827ba0564a7 into 15213684846c4f2e9df138df025f81fb(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:18:24,835 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 206af57796ad2b3d51883827ba0564a7: 2024-11-10T03:18:24,835 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., storeName=206af57796ad2b3d51883827ba0564a7/C, priority=12, startTime=1731208704771; duration=0sec 2024-11-10T03:18:24,835 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:18:24,835 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 206af57796ad2b3d51883827ba0564a7:C 2024-11-10T03:18:24,885 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 206af57796ad2b3d51883827ba0564a7 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-11-10T03:18:24,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 206af57796ad2b3d51883827ba0564a7, store=A 2024-11-10T03:18:24,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on 206af57796ad2b3d51883827ba0564a7 2024-11-10T03:18:24,885 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731208704520, max timestamp=1731208704881Num uniques 1018; 2024-11-10T03:18:24,885 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-10T03:18:24,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T03:18:24,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 206af57796ad2b3d51883827ba0564a7, store=B 2024-11-10T03:18:24,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-10T03:18:24,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 206af57796ad2b3d51883827ba0564a7, store=C 2024-11-10T03:18:24,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-10T03:18:24,893 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/A/35a29a45aa4d4d18b583de648564646f is 50, key is test_row_0/A:col10/1731208704532/Put/seqid=0 2024-11-10T03:18:24,902 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-10T03:18:24,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:24,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. as already flushing 2024-11-10T03:18:24,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:24,903 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:24,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:24,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742323_1499 (size=14541) 2024-11-10T03:18:24,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:24,906 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/A/35a29a45aa4d4d18b583de648564646f 2024-11-10T03:18:24,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-10T03:18:24,918 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/B/643dde3b39c44b07908de7bba5993b72 is 50, key is test_row_0/B:col10/1731208704532/Put/seqid=0 2024-11-10T03:18:24,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742324_1500 (size=12151) 2024-11-10T03:18:24,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:24,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46548 deadline: 1731208764946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:24,959 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:24,960 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:24,960 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:24,965 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:24,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46544 deadline: 1731208764947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:24,966 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:24,966 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:24,966 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:25,057 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-10T03:18:25,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:25,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. as already flushing 2024-11-10T03:18:25,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:25,057 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:25,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:25,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:25,072 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:25,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46548 deadline: 1731208765062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:25,072 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:25,072 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:25,073 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:25,088 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:25,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46544 deadline: 1731208765075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:25,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:25,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:25,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:25,210 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-10T03:18:25,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:25,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. as already flushing 2024-11-10T03:18:25,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:25,210 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:25,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:25,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:25,223 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/A/5693f7b5f2ec4310ada7cc9529ec32c6 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/5693f7b5f2ec4310ada7cc9529ec32c6 2024-11-10T03:18:25,230 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 206af57796ad2b3d51883827ba0564a7/A of 206af57796ad2b3d51883827ba0564a7 into 5693f7b5f2ec4310ada7cc9529ec32c6(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:18:25,230 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 206af57796ad2b3d51883827ba0564a7: 2024-11-10T03:18:25,230 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., storeName=206af57796ad2b3d51883827ba0564a7/A, priority=12, startTime=1731208704770; duration=0sec 2024-11-10T03:18:25,230 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:18:25,230 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 206af57796ad2b3d51883827ba0564a7:A 2024-11-10T03:18:25,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:25,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46548 deadline: 1731208765284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:25,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:25,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:25,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:25,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:25,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46544 deadline: 1731208765293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:25,303 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:25,303 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:25,303 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:25,338 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/B/643dde3b39c44b07908de7bba5993b72 2024-11-10T03:18:25,354 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/C/4b92c491f83443e69db1f2478e4b6bf5 is 50, key is test_row_0/C:col10/1731208704532/Put/seqid=0 2024-11-10T03:18:25,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742325_1501 (size=12151) 2024-11-10T03:18:25,362 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/C/4b92c491f83443e69db1f2478e4b6bf5 2024-11-10T03:18:25,364 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-10T03:18:25,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:25,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. as already flushing 2024-11-10T03:18:25,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:25,364 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:25,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:25,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:25,370 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/A/35a29a45aa4d4d18b583de648564646f as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/35a29a45aa4d4d18b583de648564646f 2024-11-10T03:18:25,374 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/35a29a45aa4d4d18b583de648564646f, entries=200, sequenceid=216, filesize=14.2 K 2024-11-10T03:18:25,375 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/B/643dde3b39c44b07908de7bba5993b72 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/643dde3b39c44b07908de7bba5993b72 2024-11-10T03:18:25,378 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/643dde3b39c44b07908de7bba5993b72, entries=150, sequenceid=216, filesize=11.9 K 2024-11-10T03:18:25,379 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/C/4b92c491f83443e69db1f2478e4b6bf5 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/4b92c491f83443e69db1f2478e4b6bf5 2024-11-10T03:18:25,382 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/4b92c491f83443e69db1f2478e4b6bf5, entries=150, sequenceid=216, filesize=11.9 K 2024-11-10T03:18:25,385 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~102.87 KB/105340, heapSize ~292.16 KB/299168, currentSize=73.80 KB/75570 for 206af57796ad2b3d51883827ba0564a7 in 500ms, sequenceid=216, compaction requested=false 2024-11-10T03:18:25,386 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 206af57796ad2b3d51883827ba0564a7: 2024-11-10T03:18:25,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-10T03:18:25,520 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-10T03:18:25,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:25,520 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing 206af57796ad2b3d51883827ba0564a7 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-10T03:18:25,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 206af57796ad2b3d51883827ba0564a7, store=A 2024-11-10T03:18:25,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:18:25,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 206af57796ad2b3d51883827ba0564a7, store=B 2024-11-10T03:18:25,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:18:25,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 206af57796ad2b3d51883827ba0564a7, store=C 2024-11-10T03:18:25,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:18:25,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/A/b6fe0a7994b441809eb92460cb9c9832 is 50, key is test_row_0/A:col10/1731208704922/Put/seqid=0 2024-11-10T03:18:25,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742326_1502 (size=12151) 2024-11-10T03:18:25,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on 206af57796ad2b3d51883827ba0564a7 2024-11-10T03:18:25,607 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. as already flushing 2024-11-10T03:18:25,694 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:25,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46544 deadline: 1731208765690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:25,694 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:25,694 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:25,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:25,701 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:25,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46548 deadline: 1731208765694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:25,702 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:25,702 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:25,702 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:25,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:25,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46544 deadline: 1731208765802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:25,813 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:25,813 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:25,813 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:25,821 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:25,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46548 deadline: 1731208765813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:25,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:25,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:25,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:25,939 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/A/b6fe0a7994b441809eb92460cb9c9832 2024-11-10T03:18:25,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/B/534ceb8e69504a26a9f7008678579b68 is 50, key is test_row_0/B:col10/1731208704922/Put/seqid=0 2024-11-10T03:18:25,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742327_1503 (size=12151) 2024-11-10T03:18:26,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:26,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46544 deadline: 1731208766023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:26,038 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:26,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46548 deadline: 1731208766032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:26,038 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:26,038 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:26,038 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:26,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:26,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:26,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:26,349 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:26,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46548 deadline: 1731208766343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:26,350 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:26,350 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/B/534ceb8e69504a26a9f7008678579b68 2024-11-10T03:18:26,350 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:26,350 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:26,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/C/fa7b6eb5e8ed40679a5fdc72553081e8 is 50, key is test_row_0/C:col10/1731208704922/Put/seqid=0 2024-11-10T03:18:26,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742328_1504 (size=12151) 2024-11-10T03:18:26,360 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/C/fa7b6eb5e8ed40679a5fdc72553081e8 2024-11-10T03:18:26,362 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:26,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46544 deadline: 1731208766353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:26,362 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:26,362 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:26,363 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:26,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/A/b6fe0a7994b441809eb92460cb9c9832 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/b6fe0a7994b441809eb92460cb9c9832 2024-11-10T03:18:26,368 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/b6fe0a7994b441809eb92460cb9c9832, entries=150, sequenceid=231, filesize=11.9 K 2024-11-10T03:18:26,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/B/534ceb8e69504a26a9f7008678579b68 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/534ceb8e69504a26a9f7008678579b68 2024-11-10T03:18:26,375 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/534ceb8e69504a26a9f7008678579b68, entries=150, sequenceid=231, filesize=11.9 K 2024-11-10T03:18:26,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/C/fa7b6eb5e8ed40679a5fdc72553081e8 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/fa7b6eb5e8ed40679a5fdc72553081e8 2024-11-10T03:18:26,380 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/fa7b6eb5e8ed40679a5fdc72553081e8, entries=150, sequenceid=231, filesize=11.9 K 2024-11-10T03:18:26,380 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 206af57796ad2b3d51883827ba0564a7 in 860ms, sequenceid=231, compaction requested=true 2024-11-10T03:18:26,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for 206af57796ad2b3d51883827ba0564a7: 2024-11-10T03:18:26,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:26,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-10T03:18:26,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-10T03:18:26,387 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-10T03:18:26,387 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0990 sec 2024-11-10T03:18:26,388 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 2.1030 sec 2024-11-10T03:18:26,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-10T03:18:26,432 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T03:18:26,433 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T03:18:26,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-10T03:18:26,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-10T03:18:26,436 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T03:18:26,437 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T03:18:26,437 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T03:18:26,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-10T03:18:26,589 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-10T03:18:26,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:26,589 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing 206af57796ad2b3d51883827ba0564a7 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-10T03:18:26,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 206af57796ad2b3d51883827ba0564a7, store=A 2024-11-10T03:18:26,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:18:26,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 206af57796ad2b3d51883827ba0564a7, store=B 2024-11-10T03:18:26,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:18:26,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 206af57796ad2b3d51883827ba0564a7, store=C 2024-11-10T03:18:26,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:18:26,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/A/be84602026bd49ef955a396bfc3efe65 is 50, key is test_row_0/A:col10/1731208705681/Put/seqid=0 2024-11-10T03:18:26,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742329_1505 (size=12151) 2024-11-10T03:18:26,608 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/A/be84602026bd49ef955a396bfc3efe65 2024-11-10T03:18:26,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/B/1156c1304ef044a1ac1936208d1b62f0 is 50, key is test_row_0/B:col10/1731208705681/Put/seqid=0 2024-11-10T03:18:26,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742330_1506 (size=12151) 2024-11-10T03:18:26,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-10T03:18:26,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on 206af57796ad2b3d51883827ba0564a7 2024-11-10T03:18:26,868 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. as already flushing 2024-11-10T03:18:26,917 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:26,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46544 deadline: 1731208766915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:26,918 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:26,918 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:26,918 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:26,918 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:26,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46548 deadline: 1731208766915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:26,918 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:26,918 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:26,918 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:27,024 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/B/1156c1304ef044a1ac1936208d1b62f0 2024-11-10T03:18:27,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/C/969f7bb4e3bd41c4ac1602cd719985e3 is 50, key is test_row_0/C:col10/1731208705681/Put/seqid=0 2024-11-10T03:18:27,035 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:27,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46544 deadline: 1731208767023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:27,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:27,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:27,036 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:27,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:27,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46548 deadline: 1731208767023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:27,036 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:27,036 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:27,036 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:27,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742331_1507 (size=12151) 2024-11-10T03:18:27,040 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/C/969f7bb4e3bd41c4ac1602cd719985e3 2024-11-10T03:18:27,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/A/be84602026bd49ef955a396bfc3efe65 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/be84602026bd49ef955a396bfc3efe65 2024-11-10T03:18:27,046 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/be84602026bd49ef955a396bfc3efe65, entries=150, sequenceid=249, filesize=11.9 K 2024-11-10T03:18:27,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/B/1156c1304ef044a1ac1936208d1b62f0 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/1156c1304ef044a1ac1936208d1b62f0 2024-11-10T03:18:27,049 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/1156c1304ef044a1ac1936208d1b62f0, entries=150, sequenceid=249, filesize=11.9 K 2024-11-10T03:18:27,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/C/969f7bb4e3bd41c4ac1602cd719985e3 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/969f7bb4e3bd41c4ac1602cd719985e3 2024-11-10T03:18:27,055 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/969f7bb4e3bd41c4ac1602cd719985e3, entries=150, sequenceid=249, filesize=11.9 K 2024-11-10T03:18:27,056 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 206af57796ad2b3d51883827ba0564a7 in 467ms, sequenceid=249, compaction requested=true 2024-11-10T03:18:27,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for 206af57796ad2b3d51883827ba0564a7: 2024-11-10T03:18:27,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:27,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-10T03:18:27,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-10T03:18:27,059 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-10T03:18:27,059 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 620 msec 2024-11-10T03:18:27,060 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 627 msec 2024-11-10T03:18:27,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-10T03:18:27,063 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T03:18:27,063 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T03:18:27,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-10T03:18:27,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-10T03:18:27,064 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T03:18:27,065 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T03:18:27,065 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T03:18:27,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-10T03:18:27,217 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-10T03:18:27,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:27,217 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing 206af57796ad2b3d51883827ba0564a7 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-10T03:18:27,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 206af57796ad2b3d51883827ba0564a7, store=A 2024-11-10T03:18:27,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:18:27,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 206af57796ad2b3d51883827ba0564a7, store=B 2024-11-10T03:18:27,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:18:27,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 206af57796ad2b3d51883827ba0564a7, store=C 2024-11-10T03:18:27,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:18:27,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/A/62f7c8323c3b4916b53a4961f9587649 is 50, key is test_row_0/A:col10/1731208706914/Put/seqid=0 2024-11-10T03:18:27,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742332_1508 (size=12301) 2024-11-10T03:18:27,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on 206af57796ad2b3d51883827ba0564a7 2024-11-10T03:18:27,249 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. as already flushing 2024-11-10T03:18:27,337 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:27,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46548 deadline: 1731208767326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:27,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:27,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:27,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:27,348 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:27,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46544 deadline: 1731208767337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:27,349 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:27,349 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:27,349 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:27,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-10T03:18:27,452 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:27,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46548 deadline: 1731208767443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:27,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:27,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:27,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:27,463 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:27,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46544 deadline: 1731208767453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:27,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:27,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:27,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:27,629 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/A/62f7c8323c3b4916b53a4961f9587649 2024-11-10T03:18:27,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/B/107ab47b7cdb4097812562e21cff4ec0 is 50, key is test_row_0/B:col10/1731208706914/Put/seqid=0 2024-11-10T03:18:27,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742333_1509 (size=12301) 2024-11-10T03:18:27,675 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:27,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46548 deadline: 1731208767664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:27,675 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:27,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46544 deadline: 1731208767673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:27,675 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:27,675 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:27,675 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:27,675 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:27,675 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:27,676 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:27,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-10T03:18:27,994 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:27,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46548 deadline: 1731208767983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:27,995 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:27,995 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:27,995 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:27,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46544 deadline: 1731208767983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:27,995 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:27,995 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:27,995 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:27,995 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:28,038 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/B/107ab47b7cdb4097812562e21cff4ec0 2024-11-10T03:18:28,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/C/e96e4d920ae7402e854f84322f71679a is 50, key is test_row_0/C:col10/1731208706914/Put/seqid=0 2024-11-10T03:18:28,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742334_1510 (size=12301) 2024-11-10T03:18:28,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-10T03:18:28,451 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/C/e96e4d920ae7402e854f84322f71679a 2024-11-10T03:18:28,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/A/62f7c8323c3b4916b53a4961f9587649 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/62f7c8323c3b4916b53a4961f9587649 2024-11-10T03:18:28,459 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/62f7c8323c3b4916b53a4961f9587649, entries=150, sequenceid=263, filesize=12.0 K 2024-11-10T03:18:28,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/B/107ab47b7cdb4097812562e21cff4ec0 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/107ab47b7cdb4097812562e21cff4ec0 2024-11-10T03:18:28,464 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/107ab47b7cdb4097812562e21cff4ec0, entries=150, sequenceid=263, filesize=12.0 K 2024-11-10T03:18:28,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/C/e96e4d920ae7402e854f84322f71679a as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/e96e4d920ae7402e854f84322f71679a 2024-11-10T03:18:28,469 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/e96e4d920ae7402e854f84322f71679a, entries=150, sequenceid=263, filesize=12.0 K 2024-11-10T03:18:28,469 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 206af57796ad2b3d51883827ba0564a7 in 1252ms, sequenceid=263, compaction requested=true 2024-11-10T03:18:28,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for 206af57796ad2b3d51883827ba0564a7: 2024-11-10T03:18:28,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:28,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-10T03:18:28,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-10T03:18:28,472 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-10T03:18:28,472 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4050 sec 2024-11-10T03:18:28,473 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 1.4090 sec 2024-11-10T03:18:28,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on 206af57796ad2b3d51883827ba0564a7 2024-11-10T03:18:28,512 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 206af57796ad2b3d51883827ba0564a7 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-10T03:18:28,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 206af57796ad2b3d51883827ba0564a7, store=A 2024-11-10T03:18:28,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:18:28,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 206af57796ad2b3d51883827ba0564a7, store=B 2024-11-10T03:18:28,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:18:28,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 206af57796ad2b3d51883827ba0564a7, store=C 2024-11-10T03:18:28,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:18:28,516 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/A/a1f39ec707a14b9e8439c25d05723795 is 50, key is test_row_0/A:col10/1731208708510/Put/seqid=0 2024-11-10T03:18:28,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742335_1511 (size=14741) 2024-11-10T03:18:28,521 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/A/a1f39ec707a14b9e8439c25d05723795 2024-11-10T03:18:28,538 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/B/0de5f882342b494ca95fb47d369b2d0c is 50, key is test_row_0/B:col10/1731208708510/Put/seqid=0 2024-11-10T03:18:28,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742336_1512 (size=12301) 2024-11-10T03:18:28,554 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/B/0de5f882342b494ca95fb47d369b2d0c 2024-11-10T03:18:28,560 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/C/b09350f3de7848f4b688c01ebd90ecf7 is 50, key is test_row_0/C:col10/1731208708510/Put/seqid=0 2024-11-10T03:18:28,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742337_1513 (size=12301) 2024-11-10T03:18:28,577 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/C/b09350f3de7848f4b688c01ebd90ecf7 2024-11-10T03:18:28,581 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/A/a1f39ec707a14b9e8439c25d05723795 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/a1f39ec707a14b9e8439c25d05723795 2024-11-10T03:18:28,585 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/a1f39ec707a14b9e8439c25d05723795, entries=200, sequenceid=282, filesize=14.4 K 2024-11-10T03:18:28,588 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/B/0de5f882342b494ca95fb47d369b2d0c as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/0de5f882342b494ca95fb47d369b2d0c 2024-11-10T03:18:28,592 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/0de5f882342b494ca95fb47d369b2d0c, entries=150, sequenceid=282, filesize=12.0 K 2024-11-10T03:18:28,593 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/C/b09350f3de7848f4b688c01ebd90ecf7 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/b09350f3de7848f4b688c01ebd90ecf7 2024-11-10T03:18:28,596 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/b09350f3de7848f4b688c01ebd90ecf7, entries=150, sequenceid=282, filesize=12.0 K 2024-11-10T03:18:28,597 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=60.38 KB/61830 for 206af57796ad2b3d51883827ba0564a7 in 85ms, sequenceid=282, compaction requested=true 2024-11-10T03:18:28,597 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 206af57796ad2b3d51883827ba0564a7: 2024-11-10T03:18:28,598 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-10T03:18:28,598 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 206af57796ad2b3d51883827ba0564a7:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T03:18:28,598 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:18:28,598 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-10T03:18:28,598 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 206af57796ad2b3d51883827ba0564a7:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T03:18:28,598 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:18:28,598 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 206af57796ad2b3d51883827ba0564a7:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T03:18:28,598 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T03:18:28,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on 206af57796ad2b3d51883827ba0564a7 2024-11-10T03:18:28,599 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 206af57796ad2b3d51883827ba0564a7 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-10T03:18:28,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 206af57796ad2b3d51883827ba0564a7, store=A 2024-11-10T03:18:28,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:18:28,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 206af57796ad2b3d51883827ba0564a7, store=B 2024-11-10T03:18:28,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:18:28,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 206af57796ad2b3d51883827ba0564a7, store=C 2024-11-10T03:18:28,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:18:28,602 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 73650 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-10T03:18:28,602 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 78480 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-10T03:18:28,602 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1541): 206af57796ad2b3d51883827ba0564a7/B is initiating minor compaction (all files) 2024-11-10T03:18:28,602 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1541): 206af57796ad2b3d51883827ba0564a7/A is initiating minor compaction (all files) 2024-11-10T03:18:28,602 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 206af57796ad2b3d51883827ba0564a7/B in TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:28,602 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/9c241ab08a294a7da132353cab4c57fd, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/643dde3b39c44b07908de7bba5993b72, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/534ceb8e69504a26a9f7008678579b68, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/1156c1304ef044a1ac1936208d1b62f0, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/107ab47b7cdb4097812562e21cff4ec0, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/0de5f882342b494ca95fb47d369b2d0c] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp, totalSize=71.9 K 2024-11-10T03:18:28,602 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 206af57796ad2b3d51883827ba0564a7/A in TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:28,602 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/5693f7b5f2ec4310ada7cc9529ec32c6, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/35a29a45aa4d4d18b583de648564646f, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/b6fe0a7994b441809eb92460cb9c9832, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/be84602026bd49ef955a396bfc3efe65, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/62f7c8323c3b4916b53a4961f9587649, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/a1f39ec707a14b9e8439c25d05723795] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp, totalSize=76.6 K 2024-11-10T03:18:28,603 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9c241ab08a294a7da132353cab4c57fd, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731208704301 2024-11-10T03:18:28,603 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 5693f7b5f2ec4310ada7cc9529ec32c6, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731208704301 2024-11-10T03:18:28,603 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 643dde3b39c44b07908de7bba5993b72, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731208704532 2024-11-10T03:18:28,603 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 35a29a45aa4d4d18b583de648564646f, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731208704520 2024-11-10T03:18:28,603 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 534ceb8e69504a26a9f7008678579b68, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1731208704922 2024-11-10T03:18:28,604 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1156c1304ef044a1ac1936208d1b62f0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731208705677 2024-11-10T03:18:28,604 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting b6fe0a7994b441809eb92460cb9c9832, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1731208704922 2024-11-10T03:18:28,604 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 107ab47b7cdb4097812562e21cff4ec0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1731208706902 2024-11-10T03:18:28,604 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting be84602026bd49ef955a396bfc3efe65, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731208705677 2024-11-10T03:18:28,604 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0de5f882342b494ca95fb47d369b2d0c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731208707293 2024-11-10T03:18:28,605 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/A/3e095333c68d478a9bec181daf82f0b4 is 50, key is test_row_0/A:col10/1731208708588/Put/seqid=0 2024-11-10T03:18:28,605 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 62f7c8323c3b4916b53a4961f9587649, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1731208706902 2024-11-10T03:18:28,605 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting a1f39ec707a14b9e8439c25d05723795, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731208707293 2024-11-10T03:18:28,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742338_1514 (size=14741) 2024-11-10T03:18:28,624 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 206af57796ad2b3d51883827ba0564a7#A#compaction#433 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:18:28,624 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/A/f54a46f7397b4d69999ee65e5fc71e75 is 50, key is test_row_0/A:col10/1731208708510/Put/seqid=0 2024-11-10T03:18:28,628 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 206af57796ad2b3d51883827ba0564a7#B#compaction#434 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:18:28,628 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/B/1f17baa0b11a42c59e48d2cb513e078c is 50, key is test_row_0/B:col10/1731208708510/Put/seqid=0 2024-11-10T03:18:28,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742339_1515 (size=12949) 2024-11-10T03:18:28,653 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/A/f54a46f7397b4d69999ee65e5fc71e75 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/f54a46f7397b4d69999ee65e5fc71e75 2024-11-10T03:18:28,657 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 206af57796ad2b3d51883827ba0564a7/A of 206af57796ad2b3d51883827ba0564a7 into f54a46f7397b4d69999ee65e5fc71e75(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:18:28,657 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 206af57796ad2b3d51883827ba0564a7: 2024-11-10T03:18:28,657 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., storeName=206af57796ad2b3d51883827ba0564a7/A, priority=10, startTime=1731208708598; duration=0sec 2024-11-10T03:18:28,658 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T03:18:28,658 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 206af57796ad2b3d51883827ba0564a7:A 2024-11-10T03:18:28,658 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-10T03:18:28,663 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 73650 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-10T03:18:28,663 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1541): 206af57796ad2b3d51883827ba0564a7/C is initiating minor compaction (all files) 2024-11-10T03:18:28,663 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 206af57796ad2b3d51883827ba0564a7/C in TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:28,663 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/15213684846c4f2e9df138df025f81fb, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/4b92c491f83443e69db1f2478e4b6bf5, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/fa7b6eb5e8ed40679a5fdc72553081e8, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/969f7bb4e3bd41c4ac1602cd719985e3, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/e96e4d920ae7402e854f84322f71679a, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/b09350f3de7848f4b688c01ebd90ecf7] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp, totalSize=71.9 K 2024-11-10T03:18:28,664 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 15213684846c4f2e9df138df025f81fb, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731208704301 2024-11-10T03:18:28,664 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b92c491f83443e69db1f2478e4b6bf5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731208704532 2024-11-10T03:18:28,664 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting fa7b6eb5e8ed40679a5fdc72553081e8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1731208704922 2024-11-10T03:18:28,665 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 969f7bb4e3bd41c4ac1602cd719985e3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731208705677 2024-11-10T03:18:28,665 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting e96e4d920ae7402e854f84322f71679a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1731208706902 2024-11-10T03:18:28,665 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting b09350f3de7848f4b688c01ebd90ecf7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731208707293 2024-11-10T03:18:28,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742340_1516 (size=12949) 2024-11-10T03:18:28,674 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/B/1f17baa0b11a42c59e48d2cb513e078c as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/1f17baa0b11a42c59e48d2cb513e078c 2024-11-10T03:18:28,680 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 206af57796ad2b3d51883827ba0564a7/B of 206af57796ad2b3d51883827ba0564a7 into 1f17baa0b11a42c59e48d2cb513e078c(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:18:28,680 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 206af57796ad2b3d51883827ba0564a7: 2024-11-10T03:18:28,680 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., storeName=206af57796ad2b3d51883827ba0564a7/B, priority=10, startTime=1731208708598; duration=0sec 2024-11-10T03:18:28,680 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:18:28,680 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 206af57796ad2b3d51883827ba0564a7:B 2024-11-10T03:18:28,687 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 206af57796ad2b3d51883827ba0564a7#C#compaction#435 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:18:28,688 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/C/c3699a88a174415dbd5c6a193317718e is 50, key is test_row_0/C:col10/1731208708510/Put/seqid=0 2024-11-10T03:18:28,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742341_1517 (size=12949) 2024-11-10T03:18:28,713 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/C/c3699a88a174415dbd5c6a193317718e as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/c3699a88a174415dbd5c6a193317718e 2024-11-10T03:18:28,716 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 206af57796ad2b3d51883827ba0564a7/C of 206af57796ad2b3d51883827ba0564a7 into c3699a88a174415dbd5c6a193317718e(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:18:28,716 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 206af57796ad2b3d51883827ba0564a7: 2024-11-10T03:18:28,716 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., storeName=206af57796ad2b3d51883827ba0564a7/C, priority=10, startTime=1731208708598; duration=0sec 2024-11-10T03:18:28,716 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:18:28,716 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 206af57796ad2b3d51883827ba0564a7:C 2024-11-10T03:18:28,736 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:28,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46544 deadline: 1731208768722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:28,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:28,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:28,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:28,744 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:28,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46548 deadline: 1731208768736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:28,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:28,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:28,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:28,850 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:28,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46544 deadline: 1731208768843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:28,851 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:28,851 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:28,851 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:28,864 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:28,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46548 deadline: 1731208768853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:28,864 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:28,864 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:28,864 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:29,016 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/A/3e095333c68d478a9bec181daf82f0b4 2024-11-10T03:18:29,021 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/B/463874149f764ee88e2b7bd88a889430 is 50, key is test_row_0/B:col10/1731208708588/Put/seqid=0 2024-11-10T03:18:29,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742342_1518 (size=12301) 2024-11-10T03:18:29,024 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/B/463874149f764ee88e2b7bd88a889430 2024-11-10T03:18:29,036 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/C/ae3b36192ed14b44a8ae897949fada8a is 50, key is test_row_0/C:col10/1731208708588/Put/seqid=0 2024-11-10T03:18:29,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742343_1519 (size=12301) 2024-11-10T03:18:29,070 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/C/ae3b36192ed14b44a8ae897949fada8a 2024-11-10T03:18:29,071 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:29,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46544 deadline: 1731208769062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:29,072 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:29,072 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:29,072 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:29,076 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:29,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46548 deadline: 1731208769073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:29,076 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:29,076 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:29,076 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:29,076 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/A/3e095333c68d478a9bec181daf82f0b4 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/3e095333c68d478a9bec181daf82f0b4 2024-11-10T03:18:29,081 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/3e095333c68d478a9bec181daf82f0b4, entries=200, sequenceid=295, filesize=14.4 K 2024-11-10T03:18:29,082 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/B/463874149f764ee88e2b7bd88a889430 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/463874149f764ee88e2b7bd88a889430 2024-11-10T03:18:29,085 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/463874149f764ee88e2b7bd88a889430, entries=150, sequenceid=295, filesize=12.0 K 2024-11-10T03:18:29,085 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/C/ae3b36192ed14b44a8ae897949fada8a as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/ae3b36192ed14b44a8ae897949fada8a 2024-11-10T03:18:29,094 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/ae3b36192ed14b44a8ae897949fada8a, entries=150, sequenceid=295, filesize=12.0 K 2024-11-10T03:18:29,095 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 206af57796ad2b3d51883827ba0564a7 in 496ms, sequenceid=295, compaction requested=false 2024-11-10T03:18:29,095 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 206af57796ad2b3d51883827ba0564a7: 2024-11-10T03:18:29,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-10T03:18:29,213 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T03:18:29,214 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T03:18:29,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-11-10T03:18:29,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-10T03:18:29,216 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T03:18:29,217 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T03:18:29,217 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T03:18:29,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-10T03:18:29,370 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-10T03:18:29,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:29,370 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing 206af57796ad2b3d51883827ba0564a7 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-10T03:18:29,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 206af57796ad2b3d51883827ba0564a7, store=A 2024-11-10T03:18:29,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:18:29,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 206af57796ad2b3d51883827ba0564a7, store=B 2024-11-10T03:18:29,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:18:29,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 206af57796ad2b3d51883827ba0564a7, store=C 2024-11-10T03:18:29,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:18:29,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/A/ade3de1a1f204aa99d80a824c3add6f4 is 50, key is test_row_0/A:col10/1731208708717/Put/seqid=0 2024-11-10T03:18:29,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742344_1520 (size=12301) 2024-11-10T03:18:29,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on 206af57796ad2b3d51883827ba0564a7 2024-11-10T03:18:29,387 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. as already flushing 2024-11-10T03:18:29,434 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:29,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46548 deadline: 1731208769420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:29,434 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:29,434 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:29,434 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:29,434 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:29,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46544 deadline: 1731208769421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:29,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:29,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:29,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:29,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-10T03:18:29,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:29,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46548 deadline: 1731208769543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:29,545 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:29,545 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:29,545 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:29,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:29,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46544 deadline: 1731208769543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:29,558 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:29,558 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:29,558 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:29,756 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:29,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46548 deadline: 1731208769753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:29,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:29,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:29,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:29,773 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:29,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46544 deadline: 1731208769763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:29,775 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:29,775 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:29,775 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:29,780 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/A/ade3de1a1f204aa99d80a824c3add6f4 2024-11-10T03:18:29,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/B/8261d7dd1bc44d9d8b4c00d845f83924 is 50, key is test_row_0/B:col10/1731208708717/Put/seqid=0 2024-11-10T03:18:29,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742345_1521 (size=12301) 2024-11-10T03:18:29,797 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/B/8261d7dd1bc44d9d8b4c00d845f83924 2024-11-10T03:18:29,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/C/76d5366672b743e386814ddb90a81da1 is 50, key is test_row_0/C:col10/1731208708717/Put/seqid=0 2024-11-10T03:18:29,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742346_1522 (size=12301) 2024-11-10T03:18:29,814 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/C/76d5366672b743e386814ddb90a81da1 2024-11-10T03:18:29,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/A/ade3de1a1f204aa99d80a824c3add6f4 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/ade3de1a1f204aa99d80a824c3add6f4 2024-11-10T03:18:29,822 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/ade3de1a1f204aa99d80a824c3add6f4, entries=150, sequenceid=317, filesize=12.0 K 2024-11-10T03:18:29,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/B/8261d7dd1bc44d9d8b4c00d845f83924 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/8261d7dd1bc44d9d8b4c00d845f83924 2024-11-10T03:18:29,828 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/8261d7dd1bc44d9d8b4c00d845f83924, entries=150, sequenceid=317, filesize=12.0 K 2024-11-10T03:18:29,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/C/76d5366672b743e386814ddb90a81da1 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/76d5366672b743e386814ddb90a81da1 2024-11-10T03:18:29,831 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/76d5366672b743e386814ddb90a81da1, entries=150, sequenceid=317, filesize=12.0 K 2024-11-10T03:18:29,832 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 206af57796ad2b3d51883827ba0564a7 in 462ms, sequenceid=317, compaction requested=true 2024-11-10T03:18:29,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for 206af57796ad2b3d51883827ba0564a7: 2024-11-10T03:18:29,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:29,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-11-10T03:18:29,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-11-10T03:18:29,834 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-10T03:18:29,834 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 616 msec 2024-11-10T03:18:29,837 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 621 msec 2024-11-10T03:18:29,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-10T03:18:29,844 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T03:18:29,844 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T03:18:29,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees 2024-11-10T03:18:29,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-10T03:18:29,847 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T03:18:29,848 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T03:18:29,848 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T03:18:29,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-10T03:18:30,002 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-10T03:18:30,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:30,002 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2902): Flushing 206af57796ad2b3d51883827ba0564a7 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-10T03:18:30,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 206af57796ad2b3d51883827ba0564a7, store=A 2024-11-10T03:18:30,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:18:30,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 206af57796ad2b3d51883827ba0564a7, store=B 2024-11-10T03:18:30,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:18:30,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 206af57796ad2b3d51883827ba0564a7, store=C 2024-11-10T03:18:30,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:18:30,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/A/03dd4bb8490f4d92b88e175c669e18ff is 50, key is test_row_0/A:col10/1731208709419/Put/seqid=0 2024-11-10T03:18:30,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742347_1523 (size=12301) 2024-11-10T03:18:30,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on 206af57796ad2b3d51883827ba0564a7 2024-11-10T03:18:30,068 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. as already flushing 2024-11-10T03:18:30,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-10T03:18:30,172 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:30,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46544 deadline: 1731208770163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:30,172 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:30,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46548 deadline: 1731208770164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:30,173 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:30,173 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:30,173 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:30,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:30,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:30,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:30,294 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:30,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46548 deadline: 1731208770284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:30,294 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:30,294 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:30,295 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:30,296 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:30,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46544 deadline: 1731208770284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:30,297 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:30,297 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:30,297 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:30,418 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/A/03dd4bb8490f4d92b88e175c669e18ff 2024-11-10T03:18:30,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/B/5c9b3c4d36bd476491084a0d12591821 is 50, key is test_row_0/B:col10/1731208709419/Put/seqid=0 2024-11-10T03:18:30,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742348_1524 (size=12301) 2024-11-10T03:18:30,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-10T03:18:30,513 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:30,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46548 deadline: 1731208770503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:30,514 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:30,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:30,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46544 deadline: 1731208770503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:30,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:30,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:30,514 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:30,514 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:30,514 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:30,832 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:30,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46548 deadline: 1731208770822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:30,833 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:30,833 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:30,833 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:30,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46544 deadline: 1731208770822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:30,833 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:30,833 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:30,833 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:30,833 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:30,861 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/B/5c9b3c4d36bd476491084a0d12591821 2024-11-10T03:18:30,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/C/e8658a2106fc48acbcb7935422e3951d is 50, key is test_row_0/C:col10/1731208709419/Put/seqid=0 2024-11-10T03:18:30,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742349_1525 (size=12301) 2024-11-10T03:18:30,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-10T03:18:31,275 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/C/e8658a2106fc48acbcb7935422e3951d 2024-11-10T03:18:31,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/A/03dd4bb8490f4d92b88e175c669e18ff as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/03dd4bb8490f4d92b88e175c669e18ff 2024-11-10T03:18:31,281 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/03dd4bb8490f4d92b88e175c669e18ff, entries=150, sequenceid=330, filesize=12.0 K 2024-11-10T03:18:31,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/B/5c9b3c4d36bd476491084a0d12591821 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/5c9b3c4d36bd476491084a0d12591821 2024-11-10T03:18:31,285 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/5c9b3c4d36bd476491084a0d12591821, entries=150, sequenceid=330, filesize=12.0 K 2024-11-10T03:18:31,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/C/e8658a2106fc48acbcb7935422e3951d as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/e8658a2106fc48acbcb7935422e3951d 2024-11-10T03:18:31,289 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/e8658a2106fc48acbcb7935422e3951d, entries=150, sequenceid=330, filesize=12.0 K 2024-11-10T03:18:31,290 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 206af57796ad2b3d51883827ba0564a7 in 1288ms, sequenceid=330, compaction requested=true 2024-11-10T03:18:31,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2603): Flush status journal for 206af57796ad2b3d51883827ba0564a7: 2024-11-10T03:18:31,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:31,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-11-10T03:18:31,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-11-10T03:18:31,292 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-10T03:18:31,292 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4430 sec 2024-11-10T03:18:31,294 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees in 1.4490 sec 2024-11-10T03:18:31,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on 206af57796ad2b3d51883827ba0564a7 2024-11-10T03:18:31,351 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 206af57796ad2b3d51883827ba0564a7 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-10T03:18:31,354 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731208710148, max timestamp=1731208711351Num uniques 859; 2024-11-10T03:18:31,354 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731208710148, max timestamp=1731208711351Num uniques 859; 2024-11-10T03:18:31,354 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731208710148, max timestamp=1731208711351Num uniques 859; 2024-11-10T03:18:31,354 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T03:18:31,354 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T03:18:31,354 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T03:18:31,355 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 206af57796ad2b3d51883827ba0564a7, store=A 2024-11-10T03:18:31,355 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T03:18:31,355 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 206af57796ad2b3d51883827ba0564a7, store=B 2024-11-10T03:18:31,355 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T03:18:31,355 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 206af57796ad2b3d51883827ba0564a7, store=C 2024-11-10T03:18:31,355 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T03:18:31,358 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/A/94fd751b374f4f388dda0c54ee4c44c6 is 50, key is test_row_0/A:col10/1731208711350/Put/seqid=0 2024-11-10T03:18:31,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742350_1526 (size=17181) 2024-11-10T03:18:31,443 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731208711421, max timestamp=1731208711440Num uniques 859; 2024-11-10T03:18:31,443 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731208711421, max timestamp=1731208711440Num uniques 859; 2024-11-10T03:18:31,443 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731208711421, max timestamp=1731208711440Num uniques 859; 2024-11-10T03:18:31,443 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T03:18:31,443 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T03:18:31,443 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T03:18:31,538 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731208711501, max timestamp=1731208711534Num uniques 1068; 2024-11-10T03:18:31,538 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-10T03:18:31,538 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731208711501, max timestamp=1731208711534Num uniques 1068; 2024-11-10T03:18:31,539 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-10T03:18:31,539 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731208711501, max timestamp=1731208711534Num uniques 1068; 2024-11-10T03:18:31,540 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-10T03:18:31,673 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731208711628, max timestamp=1731208711669Num uniques 1077; 2024-11-10T03:18:31,673 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731208711628, max timestamp=1731208711669Num uniques 1077; 2024-11-10T03:18:31,673 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-11-10T03:18:31,673 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731208711628, max timestamp=1731208711669Num uniques 1077; 2024-11-10T03:18:31,673 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-11-10T03:18:31,673 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-11-10T03:18:31,761 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/A/94fd751b374f4f388dda0c54ee4c44c6 2024-11-10T03:18:31,766 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/B/0db7c6b1e5ae423f8e7d8fe69f7d073a is 50, key is test_row_0/B:col10/1731208711350/Put/seqid=0 2024-11-10T03:18:31,771 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731208711746, max timestamp=1731208711768Num uniques 1036; 2024-11-10T03:18:31,771 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=10806, suffix heap size=135712,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-11-10T03:18:31,772 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731208711746, max timestamp=1731208711768Num uniques 1036; 2024-11-10T03:18:31,772 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=10806, suffix heap size=135712,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-11-10T03:18:31,772 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731208711746, max timestamp=1731208711768Num uniques 1036; 2024-11-10T03:18:31,772 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=10806, suffix heap size=135712,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-11-10T03:18:31,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742351_1527 (size=12301) 2024-11-10T03:18:31,774 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/B/0db7c6b1e5ae423f8e7d8fe69f7d073a 2024-11-10T03:18:31,780 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/C/31e775ad811f49a3bbe8f9b56e6eada8 is 50, key is test_row_0/C:col10/1731208711350/Put/seqid=0 2024-11-10T03:18:31,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742352_1528 (size=12301) 2024-11-10T03:18:31,887 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731208711846, max timestamp=1731208711884Num uniques 1095; 2024-11-10T03:18:31,887 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731208711846, max timestamp=1731208711884Num uniques 1095; 2024-11-10T03:18:31,887 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=11220, suffix heap size=141848,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=245 2024-11-10T03:18:31,888 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=11220, suffix heap size=141848,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=245 2024-11-10T03:18:31,888 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731208711846, max timestamp=1731208711884Num uniques 1095; 2024-11-10T03:18:31,888 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=11220, suffix heap size=141848,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=245 2024-11-10T03:18:32,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-10T03:18:32,003 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T03:18:32,004 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T03:18:32,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees 2024-11-10T03:18:32,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-10T03:18:32,009 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T03:18:32,010 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T03:18:32,010 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T03:18:32,040 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1731208711987, max timestamp=1731208712035Num uniques 1104; 2024-11-10T03:18:32,040 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1731208711987, max timestamp=1731208712035Num uniques 1104; 2024-11-10T03:18:32,040 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=9340, suffix heap size=142784,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=204 2024-11-10T03:18:32,040 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=9340, suffix heap size=142784,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=204 2024-11-10T03:18:32,040 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1731208711987, max timestamp=1731208712035Num uniques 1104; 2024-11-10T03:18:32,040 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=9340, suffix heap size=142784,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=204 2024-11-10T03:18:32,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-10T03:18:32,163 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1731208712136, max timestamp=1731208712160Num uniques 1063; 2024-11-10T03:18:32,163 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=7458, suffix heap size=138520,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=163 2024-11-10T03:18:32,163 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1731208712136, max timestamp=1731208712160Num uniques 1063; 2024-11-10T03:18:32,164 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=7458, suffix heap size=138520,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=163 2024-11-10T03:18:32,164 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1731208712136, max timestamp=1731208712160Num uniques 1063; 2024-11-10T03:18:32,164 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=7458, suffix heap size=138520,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=163 2024-11-10T03:18:32,165 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-10T03:18:32,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:32,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. as already flushing 2024-11-10T03:18:32,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:32,165 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:32,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:32,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:32,186 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/C/31e775ad811f49a3bbe8f9b56e6eada8 2024-11-10T03:18:32,189 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/A/94fd751b374f4f388dda0c54ee4c44c6 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/94fd751b374f4f388dda0c54ee4c44c6 2024-11-10T03:18:32,191 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/94fd751b374f4f388dda0c54ee4c44c6, entries=250, sequenceid=351, filesize=16.8 K 2024-11-10T03:18:32,192 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/B/0db7c6b1e5ae423f8e7d8fe69f7d073a as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/0db7c6b1e5ae423f8e7d8fe69f7d073a 2024-11-10T03:18:32,195 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/0db7c6b1e5ae423f8e7d8fe69f7d073a, entries=150, sequenceid=351, filesize=12.0 K 2024-11-10T03:18:32,196 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/C/31e775ad811f49a3bbe8f9b56e6eada8 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/31e775ad811f49a3bbe8f9b56e6eada8 2024-11-10T03:18:32,198 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/31e775ad811f49a3bbe8f9b56e6eada8, entries=150, sequenceid=351, filesize=12.0 K 2024-11-10T03:18:32,206 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~81.42 KB/83376, currentSize=33.54 KB/34350 for 206af57796ad2b3d51883827ba0564a7 in 855ms, sequenceid=351, compaction requested=true 2024-11-10T03:18:32,206 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 206af57796ad2b3d51883827ba0564a7: 2024-11-10T03:18:32,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 206af57796ad2b3d51883827ba0564a7:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T03:18:32,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:18:32,206 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-10T03:18:32,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 206af57796ad2b3d51883827ba0564a7:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T03:18:32,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T03:18:32,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 206af57796ad2b3d51883827ba0564a7:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T03:18:32,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-10T03:18:32,206 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-10T03:18:32,210 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 62153 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-10T03:18:32,210 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 69473 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-10T03:18:32,210 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1541): 206af57796ad2b3d51883827ba0564a7/B is initiating minor compaction (all files) 2024-11-10T03:18:32,210 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1541): 206af57796ad2b3d51883827ba0564a7/A is initiating minor compaction (all files) 2024-11-10T03:18:32,210 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 206af57796ad2b3d51883827ba0564a7/A in TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:32,210 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 206af57796ad2b3d51883827ba0564a7/B in TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:32,210 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/1f17baa0b11a42c59e48d2cb513e078c, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/463874149f764ee88e2b7bd88a889430, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/8261d7dd1bc44d9d8b4c00d845f83924, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/5c9b3c4d36bd476491084a0d12591821, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/0db7c6b1e5ae423f8e7d8fe69f7d073a] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp, totalSize=60.7 K 2024-11-10T03:18:32,210 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/f54a46f7397b4d69999ee65e5fc71e75, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/3e095333c68d478a9bec181daf82f0b4, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/ade3de1a1f204aa99d80a824c3add6f4, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/03dd4bb8490f4d92b88e175c669e18ff, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/94fd751b374f4f388dda0c54ee4c44c6] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp, totalSize=67.8 K 2024-11-10T03:18:32,210 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting f54a46f7397b4d69999ee65e5fc71e75, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731208707293 2024-11-10T03:18:32,210 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f17baa0b11a42c59e48d2cb513e078c, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731208707293 2024-11-10T03:18:32,211 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e095333c68d478a9bec181daf82f0b4, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1731208708567 2024-11-10T03:18:32,211 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 463874149f764ee88e2b7bd88a889430, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1731208708587 2024-11-10T03:18:32,211 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting ade3de1a1f204aa99d80a824c3add6f4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1731208708702 2024-11-10T03:18:32,211 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 8261d7dd1bc44d9d8b4c00d845f83924, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1731208708702 2024-11-10T03:18:32,211 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 03dd4bb8490f4d92b88e175c669e18ff, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1731208709417 2024-11-10T03:18:32,211 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c9b3c4d36bd476491084a0d12591821, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1731208709417 2024-11-10T03:18:32,211 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 94fd751b374f4f388dda0c54ee4c44c6, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1731208710148 2024-11-10T03:18:32,211 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 0db7c6b1e5ae423f8e7d8fe69f7d073a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1731208710148 2024-11-10T03:18:32,223 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 206af57796ad2b3d51883827ba0564a7 3/3 column families, dataSize=53.67 KB heapSize=145.55 KB 2024-11-10T03:18:32,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on 206af57796ad2b3d51883827ba0564a7 2024-11-10T03:18:32,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 206af57796ad2b3d51883827ba0564a7, store=A 2024-11-10T03:18:32,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T03:18:32,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 206af57796ad2b3d51883827ba0564a7, store=B 2024-11-10T03:18:32,225 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T03:18:32,225 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 206af57796ad2b3d51883827ba0564a7, store=C 2024-11-10T03:18:32,225 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T03:18:32,231 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 206af57796ad2b3d51883827ba0564a7#B#compaction#448 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:18:32,231 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 206af57796ad2b3d51883827ba0564a7#A#compaction#447 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:18:32,232 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/B/5714c65d183544a5a8480ae06a41f026 is 50, key is test_row_0/B:col10/1731208711350/Put/seqid=0 2024-11-10T03:18:32,232 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/A/6f9af81163334305b75f9a10eab587b5 is 50, key is test_row_0/A:col10/1731208711350/Put/seqid=0 2024-11-10T03:18:32,238 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/A/9b3d89f4c56d4c5da02a3dcd995465eb is 50, key is test_row_0/A:col10/1731208712219/Put/seqid=0 2024-11-10T03:18:32,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742353_1529 (size=13119) 2024-11-10T03:18:32,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742354_1530 (size=13119) 2024-11-10T03:18:32,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742355_1531 (size=17181) 2024-11-10T03:18:32,318 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-10T03:18:32,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:32,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. as already flushing 2024-11-10T03:18:32,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:32,318 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:32,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:32,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:32,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-10T03:18:32,400 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:32,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 252 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46548 deadline: 1731208772388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:32,400 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731208712322, max timestamp=1731208712397Num uniques 859; 2024-11-10T03:18:32,400 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T03:18:32,401 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:32,401 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:32,401 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731208712322, max timestamp=1731208712397Num uniques 859; 2024-11-10T03:18:32,401 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:32,401 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T03:18:32,400 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731208712322, max timestamp=1731208712397Num uniques 859; 2024-11-10T03:18:32,401 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/394d2222f724:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T03:18:32,471 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-10T03:18:32,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:32,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. as already flushing 2024-11-10T03:18:32,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:32,471 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:32,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:32,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:32,587 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:32,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 254 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46544 deadline: 1731208772567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:32,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:32,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:32,588 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:32,595 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:32,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 257 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46548 deadline: 1731208772576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:32,595 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:32,595 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:32,595 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:32,623 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-10T03:18:32,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:32,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. as already flushing 2024-11-10T03:18:32,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:32,624 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:32,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:32,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:32,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-10T03:18:32,658 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/B/5714c65d183544a5a8480ae06a41f026 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/5714c65d183544a5a8480ae06a41f026 2024-11-10T03:18:32,664 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 206af57796ad2b3d51883827ba0564a7/B of 206af57796ad2b3d51883827ba0564a7 into 5714c65d183544a5a8480ae06a41f026(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:18:32,664 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 206af57796ad2b3d51883827ba0564a7: 2024-11-10T03:18:32,664 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., storeName=206af57796ad2b3d51883827ba0564a7/B, priority=11, startTime=1731208712206; duration=0sec 2024-11-10T03:18:32,664 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T03:18:32,664 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 206af57796ad2b3d51883827ba0564a7:B 2024-11-10T03:18:32,664 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-10T03:18:32,666 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=480 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/A/9b3d89f4c56d4c5da02a3dcd995465eb 2024-11-10T03:18:32,668 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/A/6f9af81163334305b75f9a10eab587b5 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/6f9af81163334305b75f9a10eab587b5 2024-11-10T03:18:32,669 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 62153 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-10T03:18:32,669 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1541): 206af57796ad2b3d51883827ba0564a7/C is initiating minor compaction (all files) 2024-11-10T03:18:32,669 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 206af57796ad2b3d51883827ba0564a7/C in TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:32,669 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/c3699a88a174415dbd5c6a193317718e, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/ae3b36192ed14b44a8ae897949fada8a, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/76d5366672b743e386814ddb90a81da1, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/e8658a2106fc48acbcb7935422e3951d, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/31e775ad811f49a3bbe8f9b56e6eada8] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp, totalSize=60.7 K 2024-11-10T03:18:32,670 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting c3699a88a174415dbd5c6a193317718e, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731208707293 2024-11-10T03:18:32,670 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting ae3b36192ed14b44a8ae897949fada8a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1731208708587 2024-11-10T03:18:32,672 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 76d5366672b743e386814ddb90a81da1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1731208708702 2024-11-10T03:18:32,673 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting e8658a2106fc48acbcb7935422e3951d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1731208709417 2024-11-10T03:18:32,675 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/B/25d59ccf66d94e29a447cb209fdc8548 is 50, key is test_row_0/B:col10/1731208712219/Put/seqid=0 2024-11-10T03:18:32,676 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 206af57796ad2b3d51883827ba0564a7/A of 206af57796ad2b3d51883827ba0564a7 into 6f9af81163334305b75f9a10eab587b5(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:18:32,676 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 206af57796ad2b3d51883827ba0564a7: 2024-11-10T03:18:32,676 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., storeName=206af57796ad2b3d51883827ba0564a7/A, priority=11, startTime=1731208712206; duration=0sec 2024-11-10T03:18:32,676 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:18:32,676 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 206af57796ad2b3d51883827ba0564a7:A 2024-11-10T03:18:32,676 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 31e775ad811f49a3bbe8f9b56e6eada8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1731208710148 2024-11-10T03:18:32,699 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 206af57796ad2b3d51883827ba0564a7#C#compaction#451 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:18:32,700 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/C/fe8fb2ba86dd4fdc9b7124cc21950c63 is 50, key is test_row_0/C:col10/1731208711350/Put/seqid=0 2024-11-10T03:18:32,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742356_1532 (size=12301) 2024-11-10T03:18:32,706 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=480 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/B/25d59ccf66d94e29a447cb209fdc8548 2024-11-10T03:18:32,713 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:32,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 255 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46544 deadline: 1731208772694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:32,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:32,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:32,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:32,719 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:32,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 258 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46548 deadline: 1731208772703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:32,720 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:32,720 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:32,720 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:32,728 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/C/8a65cf210238477cac178fde264991c6 is 50, key is test_row_0/C:col10/1731208712219/Put/seqid=0 2024-11-10T03:18:32,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742357_1533 (size=13119) 2024-11-10T03:18:32,735 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/C/fe8fb2ba86dd4fdc9b7124cc21950c63 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/fe8fb2ba86dd4fdc9b7124cc21950c63 2024-11-10T03:18:32,748 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 206af57796ad2b3d51883827ba0564a7/C of 206af57796ad2b3d51883827ba0564a7 into fe8fb2ba86dd4fdc9b7124cc21950c63(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:18:32,748 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 206af57796ad2b3d51883827ba0564a7: 2024-11-10T03:18:32,748 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., storeName=206af57796ad2b3d51883827ba0564a7/C, priority=11, startTime=1731208712206; duration=0sec 2024-11-10T03:18:32,748 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:18:32,748 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 206af57796ad2b3d51883827ba0564a7:C 2024-11-10T03:18:32,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742358_1534 (size=12301) 2024-11-10T03:18:32,750 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=480 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/C/8a65cf210238477cac178fde264991c6 2024-11-10T03:18:32,754 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/A/9b3d89f4c56d4c5da02a3dcd995465eb as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/9b3d89f4c56d4c5da02a3dcd995465eb 2024-11-10T03:18:32,757 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/9b3d89f4c56d4c5da02a3dcd995465eb, entries=250, sequenceid=480, filesize=16.8 K 2024-11-10T03:18:32,758 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/B/25d59ccf66d94e29a447cb209fdc8548 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/25d59ccf66d94e29a447cb209fdc8548 2024-11-10T03:18:32,761 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/25d59ccf66d94e29a447cb209fdc8548, entries=150, sequenceid=480, filesize=12.0 K 2024-11-10T03:18:32,762 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/C/8a65cf210238477cac178fde264991c6 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/8a65cf210238477cac178fde264991c6 2024-11-10T03:18:32,765 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/8a65cf210238477cac178fde264991c6, entries=150, sequenceid=480, filesize=12.0 K 2024-11-10T03:18:32,766 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~165.42 KB/169392, currentSize=127.47 KB/130530 for 206af57796ad2b3d51883827ba0564a7 in 543ms, sequenceid=480, compaction requested=false 2024-11-10T03:18:32,766 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 206af57796ad2b3d51883827ba0564a7: 2024-11-10T03:18:32,777 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-10T03:18:32,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:32,777 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2902): Flushing 206af57796ad2b3d51883827ba0564a7 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-11-10T03:18:32,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 206af57796ad2b3d51883827ba0564a7, store=A 2024-11-10T03:18:32,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T03:18:32,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 206af57796ad2b3d51883827ba0564a7, store=B 2024-11-10T03:18:32,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T03:18:32,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 206af57796ad2b3d51883827ba0564a7, store=C 2024-11-10T03:18:32,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T03:18:32,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/A/48b2662dda0645179f1377ff08753b36 is 50, key is test_row_0/A:col10/1731208712508/Put/seqid=0 2024-11-10T03:18:32,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742359_1535 (size=12301) 2024-11-10T03:18:32,810 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=519 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/A/48b2662dda0645179f1377ff08753b36 2024-11-10T03:18:32,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/B/a55d5c8ada9c47b88e3a642820c6b727 is 50, key is test_row_0/B:col10/1731208712508/Put/seqid=0 2024-11-10T03:18:32,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742360_1536 (size=12301) 2024-11-10T03:18:32,844 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=519 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/B/a55d5c8ada9c47b88e3a642820c6b727 2024-11-10T03:18:32,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/C/87778dacb9334a9392798a23a25c8558 is 50, key is test_row_0/C:col10/1731208712508/Put/seqid=0 2024-11-10T03:18:32,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742361_1537 (size=12301) 2024-11-10T03:18:32,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on 206af57796ad2b3d51883827ba0564a7 2024-11-10T03:18:32,928 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. as already flushing 2024-11-10T03:18:32,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:32,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 260 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46544 deadline: 1731208772951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:32,968 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:32,968 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:32,968 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:32,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:32,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 263 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46548 deadline: 1731208772960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:32,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:32,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:32,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:33,084 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:33,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 261 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46544 deadline: 1731208773073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:33,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:33,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:33,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:33,093 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:33,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 264 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46548 deadline: 1731208773082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:33,094 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:33,094 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:33,094 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:33,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-10T03:18:33,274 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=519 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/C/87778dacb9334a9392798a23a25c8558 2024-11-10T03:18:33,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/A/48b2662dda0645179f1377ff08753b36 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/48b2662dda0645179f1377ff08753b36 2024-11-10T03:18:33,281 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/48b2662dda0645179f1377ff08753b36, entries=150, sequenceid=519, filesize=12.0 K 2024-11-10T03:18:33,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/B/a55d5c8ada9c47b88e3a642820c6b727 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/a55d5c8ada9c47b88e3a642820c6b727 2024-11-10T03:18:33,285 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/a55d5c8ada9c47b88e3a642820c6b727, entries=150, sequenceid=519, filesize=12.0 K 2024-11-10T03:18:33,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/C/87778dacb9334a9392798a23a25c8558 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/87778dacb9334a9392798a23a25c8558 2024-11-10T03:18:33,287 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/87778dacb9334a9392798a23a25c8558, entries=150, sequenceid=519, filesize=12.0 K 2024-11-10T03:18:33,288 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for 206af57796ad2b3d51883827ba0564a7 in 511ms, sequenceid=519, compaction requested=true 2024-11-10T03:18:33,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2603): Flush status journal for 206af57796ad2b3d51883827ba0564a7: 2024-11-10T03:18:33,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:33,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=138 2024-11-10T03:18:33,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4169): Remote procedure done, pid=138 2024-11-10T03:18:33,291 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-10T03:18:33,291 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2790 sec 2024-11-10T03:18:33,292 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees in 1.2870 sec 2024-11-10T03:18:33,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on 206af57796ad2b3d51883827ba0564a7 2024-11-10T03:18:33,300 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 206af57796ad2b3d51883827ba0564a7 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-10T03:18:33,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 206af57796ad2b3d51883827ba0564a7, store=A 2024-11-10T03:18:33,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:18:33,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 206af57796ad2b3d51883827ba0564a7, store=B 2024-11-10T03:18:33,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:18:33,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 206af57796ad2b3d51883827ba0564a7, store=C 2024-11-10T03:18:33,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:18:33,305 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/A/8cce2fc8bc03498ab7089cab49637b0f is 50, key is test_row_0/A:col10/1731208713299/Put/seqid=0 2024-11-10T03:18:33,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742362_1538 (size=14741) 2024-11-10T03:18:33,387 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:33,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 272 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46544 deadline: 1731208773382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:33,387 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:33,387 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:33,387 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:33,396 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:33,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 273 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46548 deadline: 1731208773387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:33,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:33,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:33,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:33,401 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:33,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46562 deadline: 1731208773393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:33,402 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 394d2222f724,38139,1731208572054 for 'test_row_1' in 206af57796ad2b3d51883827ba0564a7 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18307 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T03:18:33,403 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:33,403 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:33,403 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:33,441 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:33,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46600 deadline: 1731208773433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:33,442 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 394d2222f724,38139,1731208572054 for 'test_row_0' in 206af57796ad2b3d51883827ba0564a7 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18352 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T03:18:33,442 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:33,443 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:33,443 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:33,450 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:33,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46608 deadline: 1731208773443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:33,452 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 394d2222f724,38139,1731208572054 for 'test_row_0' in 206af57796ad2b3d51883827ba0564a7 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18358 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T03:18:33,452 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:33,452 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:33,452 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:33,503 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:33,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 273 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46544 deadline: 1731208773493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:33,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:33,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:33,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:33,509 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:33,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 274 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46548 deadline: 1731208773503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:33,509 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:33,510 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:33,510 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:33,714 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:33,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 274 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46544 deadline: 1731208773713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:33,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:33,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 275 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46548 deadline: 1731208773714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:33,715 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:33,715 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:33,715 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:33,716 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:33,716 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:33,716 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:33,720 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=531 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/A/8cce2fc8bc03498ab7089cab49637b0f 2024-11-10T03:18:33,725 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/B/6f05b943a3c442e49fc80eaf81ab92ca is 50, key is test_row_0/B:col10/1731208713299/Put/seqid=0 2024-11-10T03:18:33,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742363_1539 (size=12301) 2024-11-10T03:18:33,733 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=531 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/B/6f05b943a3c442e49fc80eaf81ab92ca 2024-11-10T03:18:33,738 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/C/d23f345ed3744c8eb283e7921d1db507 is 50, key is test_row_0/C:col10/1731208713299/Put/seqid=0 2024-11-10T03:18:33,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742364_1540 (size=12301) 2024-11-10T03:18:33,745 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=531 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/C/d23f345ed3744c8eb283e7921d1db507 2024-11-10T03:18:33,755 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/A/8cce2fc8bc03498ab7089cab49637b0f as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/8cce2fc8bc03498ab7089cab49637b0f 2024-11-10T03:18:33,761 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/8cce2fc8bc03498ab7089cab49637b0f, entries=200, sequenceid=531, filesize=14.4 K 2024-11-10T03:18:33,762 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/B/6f05b943a3c442e49fc80eaf81ab92ca as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/6f05b943a3c442e49fc80eaf81ab92ca 2024-11-10T03:18:33,765 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/6f05b943a3c442e49fc80eaf81ab92ca, entries=150, sequenceid=531, filesize=12.0 K 2024-11-10T03:18:33,766 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/C/d23f345ed3744c8eb283e7921d1db507 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/d23f345ed3744c8eb283e7921d1db507 2024-11-10T03:18:33,774 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/d23f345ed3744c8eb283e7921d1db507, entries=150, sequenceid=531, filesize=12.0 K 2024-11-10T03:18:33,776 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 206af57796ad2b3d51883827ba0564a7 in 476ms, sequenceid=531, compaction requested=true 2024-11-10T03:18:33,776 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 206af57796ad2b3d51883827ba0564a7: 2024-11-10T03:18:33,777 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T03:18:33,777 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 206af57796ad2b3d51883827ba0564a7:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T03:18:33,777 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:18:33,777 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T03:18:33,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 206af57796ad2b3d51883827ba0564a7:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T03:18:33,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:18:33,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 206af57796ad2b3d51883827ba0564a7:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T03:18:33,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T03:18:33,780 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 57342 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T03:18:33,780 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1541): 206af57796ad2b3d51883827ba0564a7/A is initiating minor compaction (all files) 2024-11-10T03:18:33,780 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 206af57796ad2b3d51883827ba0564a7/A in TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:33,780 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/6f9af81163334305b75f9a10eab587b5, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/9b3d89f4c56d4c5da02a3dcd995465eb, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/48b2662dda0645179f1377ff08753b36, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/8cce2fc8bc03498ab7089cab49637b0f] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp, totalSize=56.0 K 2024-11-10T03:18:33,781 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50022 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T03:18:33,781 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6f9af81163334305b75f9a10eab587b5, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1731208710148 2024-11-10T03:18:33,781 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1541): 206af57796ad2b3d51883827ba0564a7/B is initiating minor compaction (all files) 2024-11-10T03:18:33,781 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 206af57796ad2b3d51883827ba0564a7/B in TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:33,781 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/5714c65d183544a5a8480ae06a41f026, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/25d59ccf66d94e29a447cb209fdc8548, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/a55d5c8ada9c47b88e3a642820c6b727, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/6f05b943a3c442e49fc80eaf81ab92ca] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp, totalSize=48.8 K 2024-11-10T03:18:33,781 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b3d89f4c56d4c5da02a3dcd995465eb, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=480, earliestPutTs=1731208712160 2024-11-10T03:18:33,781 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 5714c65d183544a5a8480ae06a41f026, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1731208710148 2024-11-10T03:18:33,781 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 48b2662dda0645179f1377ff08753b36, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=519, earliestPutTs=1731208712508 2024-11-10T03:18:33,781 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 25d59ccf66d94e29a447cb209fdc8548, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=480, earliestPutTs=1731208712183 2024-11-10T03:18:33,782 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8cce2fc8bc03498ab7089cab49637b0f, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=531, earliestPutTs=1731208712935 2024-11-10T03:18:33,782 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting a55d5c8ada9c47b88e3a642820c6b727, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=519, earliestPutTs=1731208712508 2024-11-10T03:18:33,783 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f05b943a3c442e49fc80eaf81ab92ca, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=531, earliestPutTs=1731208712950 2024-11-10T03:18:33,797 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 206af57796ad2b3d51883827ba0564a7#A#compaction#459 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:18:33,798 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/A/fab9c9223eed4c9cbb76424fbcb4d10b is 50, key is test_row_0/A:col10/1731208713299/Put/seqid=0 2024-11-10T03:18:33,811 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 206af57796ad2b3d51883827ba0564a7#B#compaction#460 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:18:33,812 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/B/15c4db04306e4c558050dc870111aceb is 50, key is test_row_0/B:col10/1731208713299/Put/seqid=0 2024-11-10T03:18:33,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742365_1541 (size=13255) 2024-11-10T03:18:33,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742366_1542 (size=13255) 2024-11-10T03:18:34,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on 206af57796ad2b3d51883827ba0564a7 2024-11-10T03:18:34,028 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 206af57796ad2b3d51883827ba0564a7 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-10T03:18:34,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 206af57796ad2b3d51883827ba0564a7, store=A 2024-11-10T03:18:34,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T03:18:34,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 206af57796ad2b3d51883827ba0564a7, store=B 2024-11-10T03:18:34,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T03:18:34,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 206af57796ad2b3d51883827ba0564a7, store=C 2024-11-10T03:18:34,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T03:18:34,033 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/A/16a241386396466dab94bc9eb5f8b782 is 50, key is test_row_0/A:col10/1731208714027/Put/seqid=0 2024-11-10T03:18:34,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742367_1543 (size=14741) 2024-11-10T03:18:34,073 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:34,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 279 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46544 deadline: 1731208774067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:34,073 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:34,073 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:34,073 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:34,092 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:34,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 281 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46548 deadline: 1731208774074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:34,093 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:34,093 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:34,093 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:34,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-10T03:18:34,153 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T03:18:34,153 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T03:18:34,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees 2024-11-10T03:18:34,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-10T03:18:34,156 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T03:18:34,159 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T03:18:34,159 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T03:18:34,196 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:34,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 280 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46544 deadline: 1731208774183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:34,196 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:34,197 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:34,197 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:34,214 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:34,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 282 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46548 deadline: 1731208774205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:34,215 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:34,215 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:34,215 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:34,249 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/A/fab9c9223eed4c9cbb76424fbcb4d10b as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/fab9c9223eed4c9cbb76424fbcb4d10b 2024-11-10T03:18:34,263 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/B/15c4db04306e4c558050dc870111aceb as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/15c4db04306e4c558050dc870111aceb 2024-11-10T03:18:34,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-10T03:18:34,280 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 206af57796ad2b3d51883827ba0564a7/A of 206af57796ad2b3d51883827ba0564a7 into fab9c9223eed4c9cbb76424fbcb4d10b(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:18:34,280 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 206af57796ad2b3d51883827ba0564a7: 2024-11-10T03:18:34,280 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., storeName=206af57796ad2b3d51883827ba0564a7/A, priority=12, startTime=1731208713777; duration=0sec 2024-11-10T03:18:34,281 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T03:18:34,281 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 206af57796ad2b3d51883827ba0564a7:A 2024-11-10T03:18:34,281 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T03:18:34,286 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 206af57796ad2b3d51883827ba0564a7/B of 206af57796ad2b3d51883827ba0564a7 into 15c4db04306e4c558050dc870111aceb(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:18:34,286 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 206af57796ad2b3d51883827ba0564a7: 2024-11-10T03:18:34,286 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., storeName=206af57796ad2b3d51883827ba0564a7/B, priority=12, startTime=1731208713777; duration=0sec 2024-11-10T03:18:34,286 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:18:34,286 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 206af57796ad2b3d51883827ba0564a7:B 2024-11-10T03:18:34,286 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50022 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T03:18:34,287 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1541): 206af57796ad2b3d51883827ba0564a7/C is initiating minor compaction (all files) 2024-11-10T03:18:34,287 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 206af57796ad2b3d51883827ba0564a7/C in TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:34,287 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/fe8fb2ba86dd4fdc9b7124cc21950c63, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/8a65cf210238477cac178fde264991c6, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/87778dacb9334a9392798a23a25c8558, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/d23f345ed3744c8eb283e7921d1db507] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp, totalSize=48.8 K 2024-11-10T03:18:34,289 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting fe8fb2ba86dd4fdc9b7124cc21950c63, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1731208710148 2024-11-10T03:18:34,289 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a65cf210238477cac178fde264991c6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=480, earliestPutTs=1731208712183 2024-11-10T03:18:34,290 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 87778dacb9334a9392798a23a25c8558, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=519, earliestPutTs=1731208712508 2024-11-10T03:18:34,290 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting d23f345ed3744c8eb283e7921d1db507, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=531, earliestPutTs=1731208712950 2024-11-10T03:18:34,312 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-10T03:18:34,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:34,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. as already flushing 2024-11-10T03:18:34,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:34,312 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:34,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:34,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:34,317 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 206af57796ad2b3d51883827ba0564a7#C#compaction#462 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:18:34,317 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/C/69b6067e89144899ad656c3d24c0e509 is 50, key is test_row_0/C:col10/1731208713299/Put/seqid=0 2024-11-10T03:18:34,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742368_1544 (size=13255) 2024-11-10T03:18:34,357 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/C/69b6067e89144899ad656c3d24c0e509 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/69b6067e89144899ad656c3d24c0e509 2024-11-10T03:18:34,361 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 206af57796ad2b3d51883827ba0564a7/C of 206af57796ad2b3d51883827ba0564a7 into 69b6067e89144899ad656c3d24c0e509(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:18:34,361 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 206af57796ad2b3d51883827ba0564a7: 2024-11-10T03:18:34,361 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., storeName=206af57796ad2b3d51883827ba0564a7/C, priority=12, startTime=1731208713779; duration=0sec 2024-11-10T03:18:34,361 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:18:34,361 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 206af57796ad2b3d51883827ba0564a7:C 2024-11-10T03:18:34,363 INFO [Thread-2055 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2055. 2024-11-10T03:18:34,363 DEBUG [Thread-2055 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T03:18:34,363 DEBUG [Thread-2055 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:18:34,363 DEBUG [Thread-2055 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:18:34,363 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T03:18:34,364 INFO [Thread-2049 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2049. 2024-11-10T03:18:34,364 DEBUG [Thread-2049 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T03:18:34,364 DEBUG [Thread-2049 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:18:34,364 DEBUG [Thread-2049 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:18:34,365 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T03:18:34,370 INFO [Thread-2053 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2053. 2024-11-10T03:18:34,370 DEBUG [Thread-2053 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T03:18:34,370 DEBUG [Thread-2053 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:18:34,370 DEBUG [Thread-2053 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:18:34,370 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T03:18:34,375 INFO [Thread-2057 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2057. 2024-11-10T03:18:34,375 DEBUG [Thread-2057 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T03:18:34,375 DEBUG [Thread-2057 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:18:34,375 DEBUG [Thread-2057 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:18:34,375 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T03:18:34,378 INFO [Thread-2051 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2051. 2024-11-10T03:18:34,378 DEBUG [Thread-2051 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T03:18:34,378 DEBUG [Thread-2051 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:18:34,378 DEBUG [Thread-2051 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:18:34,378 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T03:18:34,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:34,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 281 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46544 deadline: 1731208774403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:34,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:34,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:34,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:34,423 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:34,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 283 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46548 deadline: 1731208774423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:34,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:34,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:34,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:34,437 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=552 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/A/16a241386396466dab94bc9eb5f8b782 2024-11-10T03:18:34,442 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/B/f085536494b44a0697657857547148d8 is 50, key is test_row_0/B:col10/1731208714027/Put/seqid=0 2024-11-10T03:18:34,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742369_1545 (size=12301) 2024-11-10T03:18:34,466 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-10T03:18:34,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:34,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. as already flushing 2024-11-10T03:18:34,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:34,466 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:34,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:34,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:34,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-10T03:18:34,619 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-10T03:18:34,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:34,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. as already flushing 2024-11-10T03:18:34,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:34,619 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:34,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:34,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:34,713 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:34,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 282 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46544 deadline: 1731208774713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:34,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:34,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:34,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:34,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:34,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 284 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46548 deadline: 1731208774733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:34,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:34,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:34,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:34,771 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-10T03:18:34,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:34,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. as already flushing 2024-11-10T03:18:34,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:34,772 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:34,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:34,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:34,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-10T03:18:34,846 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=552 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/B/f085536494b44a0697657857547148d8 2024-11-10T03:18:34,851 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/C/368156794de94d3f890ec7263eb64d61 is 50, key is test_row_0/C:col10/1731208714027/Put/seqid=0 2024-11-10T03:18:34,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742370_1546 (size=12301) 2024-11-10T03:18:34,925 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-10T03:18:34,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:34,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. as already flushing 2024-11-10T03:18:34,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:34,925 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:34,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:34,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:35,077 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-10T03:18:35,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:35,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. as already flushing 2024-11-10T03:18:35,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:35,077 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:35,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:35,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:35,223 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:35,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 283 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46544 deadline: 1731208775223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:35,223 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:35,223 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:35,223 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:35,229 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-10T03:18:35,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:35,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. as already flushing 2024-11-10T03:18:35,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:35,230 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:35,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:35,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:35,243 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:35,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 285 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46548 deadline: 1731208775243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:35,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 , the old value is region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:35,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:35,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., hostname=394d2222f724,38139,1731208572054, seqNum=2 because the exception is null or not the one we care about 2024-11-10T03:18:35,254 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=552 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/C/368156794de94d3f890ec7263eb64d61 2024-11-10T03:18:35,258 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/A/16a241386396466dab94bc9eb5f8b782 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/16a241386396466dab94bc9eb5f8b782 2024-11-10T03:18:35,261 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/16a241386396466dab94bc9eb5f8b782, entries=200, sequenceid=552, filesize=14.4 K 2024-11-10T03:18:35,261 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/B/f085536494b44a0697657857547148d8 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/f085536494b44a0697657857547148d8 2024-11-10T03:18:35,265 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/f085536494b44a0697657857547148d8, entries=150, sequenceid=552, filesize=12.0 K 2024-11-10T03:18:35,265 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/C/368156794de94d3f890ec7263eb64d61 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/368156794de94d3f890ec7263eb64d61 2024-11-10T03:18:35,270 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/368156794de94d3f890ec7263eb64d61, entries=150, sequenceid=552, filesize=12.0 K 2024-11-10T03:18:35,270 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 206af57796ad2b3d51883827ba0564a7 in 1242ms, sequenceid=552, compaction requested=false 2024-11-10T03:18:35,270 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 206af57796ad2b3d51883827ba0564a7: 2024-11-10T03:18:35,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-10T03:18:35,382 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-10T03:18:35,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:35,383 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2902): Flushing 206af57796ad2b3d51883827ba0564a7 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-10T03:18:35,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 206af57796ad2b3d51883827ba0564a7, store=A 2024-11-10T03:18:35,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:18:35,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 206af57796ad2b3d51883827ba0564a7, store=B 2024-11-10T03:18:35,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:18:35,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 206af57796ad2b3d51883827ba0564a7, store=C 2024-11-10T03:18:35,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:18:35,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/A/273ce4ce912a43bfa7c415fc1db3c2de is 50, key is test_row_0/A:col10/1731208714070/Put/seqid=0 2024-11-10T03:18:35,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742371_1547 (size=12301) 2024-11-10T03:18:35,790 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=566 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/A/273ce4ce912a43bfa7c415fc1db3c2de 2024-11-10T03:18:35,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/B/d9c0855f5eb044eda08db2005a400998 is 50, key is test_row_0/B:col10/1731208714070/Put/seqid=0 2024-11-10T03:18:35,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742372_1548 (size=12301) 2024-11-10T03:18:36,199 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=566 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/B/d9c0855f5eb044eda08db2005a400998 2024-11-10T03:18:36,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/C/6f4fe489ebb84341b1a1ac08bbc10452 is 50, key is test_row_0/C:col10/1731208714070/Put/seqid=0 2024-11-10T03:18:36,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742373_1549 (size=12301) 2024-11-10T03:18:36,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on 206af57796ad2b3d51883827ba0564a7 2024-11-10T03:18:36,234 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. as already flushing 2024-11-10T03:18:36,234 INFO [Thread-2040 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2040. 2024-11-10T03:18:36,234 DEBUG [Thread-2040 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T03:18:36,234 DEBUG [Thread-2040 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:18:36,234 DEBUG [Thread-2040 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:18:36,234 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T03:18:36,255 INFO [Thread-2046 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2046. 2024-11-10T03:18:36,255 DEBUG [Thread-2046 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T03:18:36,255 DEBUG [Thread-2046 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:18:36,255 DEBUG [Thread-2046 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:18:36,255 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T03:18:36,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-10T03:18:36,607 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=566 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/C/6f4fe489ebb84341b1a1ac08bbc10452 2024-11-10T03:18:36,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/A/273ce4ce912a43bfa7c415fc1db3c2de as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/273ce4ce912a43bfa7c415fc1db3c2de 2024-11-10T03:18:36,613 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/273ce4ce912a43bfa7c415fc1db3c2de, entries=150, sequenceid=566, filesize=12.0 K 2024-11-10T03:18:36,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/B/d9c0855f5eb044eda08db2005a400998 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/d9c0855f5eb044eda08db2005a400998 2024-11-10T03:18:36,615 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/d9c0855f5eb044eda08db2005a400998, entries=150, sequenceid=566, filesize=12.0 K 2024-11-10T03:18:36,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/C/6f4fe489ebb84341b1a1ac08bbc10452 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/6f4fe489ebb84341b1a1ac08bbc10452 2024-11-10T03:18:36,618 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/6f4fe489ebb84341b1a1ac08bbc10452, entries=150, sequenceid=566, filesize=12.0 K 2024-11-10T03:18:36,619 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=13.42 KB/13740 for 206af57796ad2b3d51883827ba0564a7 in 1237ms, sequenceid=566, compaction requested=true 2024-11-10T03:18:36,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2603): Flush status journal for 206af57796ad2b3d51883827ba0564a7: 2024-11-10T03:18:36,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:36,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=140 2024-11-10T03:18:36,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4169): Remote procedure done, pid=140 2024-11-10T03:18:36,621 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-10T03:18:36,621 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4610 sec 2024-11-10T03:18:36,622 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees in 2.4670 sec 2024-11-10T03:18:38,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-10T03:18:38,313 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T03:18:40,312 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-10T03:18:43,454 INFO [Thread-2044 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2044. 2024-11-10T03:18:43,454 DEBUG [Thread-2044 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T03:18:43,454 DEBUG [Thread-2044 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:18:43,454 DEBUG [Thread-2044 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:18:43,454 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T03:18:43,485 INFO [Thread-2042 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2042. 2024-11-10T03:18:43,485 DEBUG [Thread-2042 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T03:18:43,485 DEBUG [Thread-2042 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:18:43,485 DEBUG [Thread-2042 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:18:43,488 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T03:18:43,545 INFO [Thread-2038 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2038. 2024-11-10T03:18:43,545 DEBUG [Thread-2038 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T03:18:43,545 DEBUG [Thread-2038 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:18:43,545 DEBUG [Thread-2038 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:18:43,545 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T03:18:43,545 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-10T03:18:43,545 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 9 2024-11-10T03:18:43,545 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 221 2024-11-10T03:18:43,545 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 13 2024-11-10T03:18:43,545 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 12 2024-11-10T03:18:43,545 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 222 2024-11-10T03:18:43,545 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-10T03:18:43,545 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-10T03:18:43,545 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1012 2024-11-10T03:18:43,545 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3034 rows 2024-11-10T03:18:43,545 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1021 2024-11-10T03:18:43,545 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3057 rows 2024-11-10T03:18:43,545 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1013 2024-11-10T03:18:43,545 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3039 rows 2024-11-10T03:18:43,545 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1016 2024-11-10T03:18:43,545 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3047 rows 2024-11-10T03:18:43,545 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1020 2024-11-10T03:18:43,545 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3058 rows 2024-11-10T03:18:43,545 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T03:18:43,546 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T03:18:43,546 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:18:43,546 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:18:43,546 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T03:18:43,546 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-10T03:18:43,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-10T03:18:43,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-10T03:18:43,549 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731208723549"}]},"ts":"1731208723549"} 2024-11-10T03:18:43,550 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-10T03:18:43,550 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-10T03:18:43,550 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-10T03:18:43,552 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=206af57796ad2b3d51883827ba0564a7, UNASSIGN}] 2024-11-10T03:18:43,552 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=206af57796ad2b3d51883827ba0564a7, UNASSIGN 2024-11-10T03:18:43,553 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=206af57796ad2b3d51883827ba0564a7, regionState=CLOSING, regionLocation=394d2222f724,38139,1731208572054 2024-11-10T03:18:43,554 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=206af57796ad2b3d51883827ba0564a7, UNASSIGN because future has completed 2024-11-10T03:18:43,554 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-10T03:18:43,554 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE, hasLock=false; CloseRegionProcedure 206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054}] 2024-11-10T03:18:43,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-10T03:18:43,706 INFO [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(122): Close 206af57796ad2b3d51883827ba0564a7 2024-11-10T03:18:43,706 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-10T03:18:43,706 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1722): Closing 206af57796ad2b3d51883827ba0564a7, disabling compactions & flushes 2024-11-10T03:18:43,707 INFO [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:43,707 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:43,707 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. after waiting 0 ms 2024-11-10T03:18:43,707 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:43,707 INFO [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(2902): Flushing 206af57796ad2b3d51883827ba0564a7 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-10T03:18:43,707 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 206af57796ad2b3d51883827ba0564a7, store=A 2024-11-10T03:18:43,707 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:18:43,707 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 206af57796ad2b3d51883827ba0564a7, store=B 2024-11-10T03:18:43,707 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:18:43,707 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 206af57796ad2b3d51883827ba0564a7, store=C 2024-11-10T03:18:43,707 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:18:43,711 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/A/787b3eb5ccb64766b27875d6d5ccbba6 is 50, key is test_row_0/A:col10/1731208723543/Put/seqid=0 2024-11-10T03:18:43,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742374_1550 (size=12301) 2024-11-10T03:18:43,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-10T03:18:44,114 INFO [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=574 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/A/787b3eb5ccb64766b27875d6d5ccbba6 2024-11-10T03:18:44,120 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/B/5d7349365190482385981bb8b0fc0e27 is 50, key is test_row_0/B:col10/1731208723543/Put/seqid=0 2024-11-10T03:18:44,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742375_1551 (size=12301) 2024-11-10T03:18:44,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-10T03:18:44,523 INFO [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=574 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/B/5d7349365190482385981bb8b0fc0e27 2024-11-10T03:18:44,533 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/C/fd1ab996003241ca92f52b63667b2dd5 is 50, key is test_row_0/C:col10/1731208723543/Put/seqid=0 2024-11-10T03:18:44,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742376_1552 (size=12301) 2024-11-10T03:18:44,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-10T03:18:44,938 INFO [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=574 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/C/fd1ab996003241ca92f52b63667b2dd5 2024-11-10T03:18:44,941 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/A/787b3eb5ccb64766b27875d6d5ccbba6 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/787b3eb5ccb64766b27875d6d5ccbba6 2024-11-10T03:18:44,944 INFO [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/787b3eb5ccb64766b27875d6d5ccbba6, entries=150, sequenceid=574, filesize=12.0 K 2024-11-10T03:18:44,944 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/B/5d7349365190482385981bb8b0fc0e27 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/5d7349365190482385981bb8b0fc0e27 2024-11-10T03:18:44,946 INFO [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/5d7349365190482385981bb8b0fc0e27, entries=150, sequenceid=574, filesize=12.0 K 2024-11-10T03:18:44,947 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/.tmp/C/fd1ab996003241ca92f52b63667b2dd5 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/fd1ab996003241ca92f52b63667b2dd5 2024-11-10T03:18:44,949 INFO [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/fd1ab996003241ca92f52b63667b2dd5, entries=150, sequenceid=574, filesize=12.0 K 2024-11-10T03:18:44,949 INFO [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 206af57796ad2b3d51883827ba0564a7 in 1242ms, sequenceid=574, compaction requested=true 2024-11-10T03:18:44,950 DEBUG [StoreCloser-TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/acaf286142e44661a6c8898a5be63bba, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/5c80644bf9e34688bfd6e319a7c12aa2, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/6b5595352c3b45bcadcbeabe7b80c93e, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/b737e77fc4884dfbab73048fa2a913d7, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/ca1c182e3834440699310febb62baa93, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/e3a5cde597df41acb2c0a7ad8c21474d, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/0a6dc3cda8664244be5f6e19869248af, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/57e3d38f7be1476c9d28596bd2a5c0ca, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/37e0da99c40042f6b4339618acf551f7, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/617729d416464d458a31e69855fd0071, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/be333081c3dd47d0bdbc1d0ee19ffb9a, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/e7a0d9198a6a4b4fb39a71973f4444d2, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/27d4d992f2394073bb5067300066221b, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/5693f7b5f2ec4310ada7cc9529ec32c6, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/35a29a45aa4d4d18b583de648564646f, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/b6fe0a7994b441809eb92460cb9c9832, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/be84602026bd49ef955a396bfc3efe65, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/62f7c8323c3b4916b53a4961f9587649, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/a1f39ec707a14b9e8439c25d05723795, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/f54a46f7397b4d69999ee65e5fc71e75, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/3e095333c68d478a9bec181daf82f0b4, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/ade3de1a1f204aa99d80a824c3add6f4, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/03dd4bb8490f4d92b88e175c669e18ff, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/94fd751b374f4f388dda0c54ee4c44c6, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/6f9af81163334305b75f9a10eab587b5, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/9b3d89f4c56d4c5da02a3dcd995465eb, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/48b2662dda0645179f1377ff08753b36, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/8cce2fc8bc03498ab7089cab49637b0f] to archive 2024-11-10T03:18:44,951 DEBUG [StoreCloser-TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T03:18:44,952 DEBUG [StoreCloser-TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/acaf286142e44661a6c8898a5be63bba to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/acaf286142e44661a6c8898a5be63bba 2024-11-10T03:18:44,952 DEBUG [StoreCloser-TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/5c80644bf9e34688bfd6e319a7c12aa2 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/5c80644bf9e34688bfd6e319a7c12aa2 2024-11-10T03:18:44,953 DEBUG [StoreCloser-TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/6b5595352c3b45bcadcbeabe7b80c93e to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/6b5595352c3b45bcadcbeabe7b80c93e 2024-11-10T03:18:44,954 DEBUG [StoreCloser-TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/b737e77fc4884dfbab73048fa2a913d7 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/b737e77fc4884dfbab73048fa2a913d7 2024-11-10T03:18:44,955 DEBUG [StoreCloser-TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/ca1c182e3834440699310febb62baa93 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/ca1c182e3834440699310febb62baa93 2024-11-10T03:18:44,956 DEBUG [StoreCloser-TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/e3a5cde597df41acb2c0a7ad8c21474d to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/e3a5cde597df41acb2c0a7ad8c21474d 2024-11-10T03:18:44,956 DEBUG [StoreCloser-TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/0a6dc3cda8664244be5f6e19869248af to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/0a6dc3cda8664244be5f6e19869248af 2024-11-10T03:18:44,957 DEBUG [StoreCloser-TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/57e3d38f7be1476c9d28596bd2a5c0ca to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/57e3d38f7be1476c9d28596bd2a5c0ca 2024-11-10T03:18:44,958 DEBUG [StoreCloser-TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/37e0da99c40042f6b4339618acf551f7 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/37e0da99c40042f6b4339618acf551f7 2024-11-10T03:18:44,959 DEBUG [StoreCloser-TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/617729d416464d458a31e69855fd0071 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/617729d416464d458a31e69855fd0071 2024-11-10T03:18:44,959 DEBUG [StoreCloser-TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/be333081c3dd47d0bdbc1d0ee19ffb9a to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/be333081c3dd47d0bdbc1d0ee19ffb9a 2024-11-10T03:18:44,960 DEBUG [StoreCloser-TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/e7a0d9198a6a4b4fb39a71973f4444d2 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/e7a0d9198a6a4b4fb39a71973f4444d2 2024-11-10T03:18:44,961 DEBUG [StoreCloser-TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/27d4d992f2394073bb5067300066221b to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/27d4d992f2394073bb5067300066221b 2024-11-10T03:18:44,961 DEBUG [StoreCloser-TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/5693f7b5f2ec4310ada7cc9529ec32c6 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/5693f7b5f2ec4310ada7cc9529ec32c6 2024-11-10T03:18:44,962 DEBUG [StoreCloser-TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/35a29a45aa4d4d18b583de648564646f to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/35a29a45aa4d4d18b583de648564646f 2024-11-10T03:18:44,963 DEBUG [StoreCloser-TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/b6fe0a7994b441809eb92460cb9c9832 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/b6fe0a7994b441809eb92460cb9c9832 2024-11-10T03:18:44,963 DEBUG [StoreCloser-TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/be84602026bd49ef955a396bfc3efe65 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/be84602026bd49ef955a396bfc3efe65 2024-11-10T03:18:44,964 DEBUG [StoreCloser-TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/62f7c8323c3b4916b53a4961f9587649 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/62f7c8323c3b4916b53a4961f9587649 2024-11-10T03:18:44,965 DEBUG [StoreCloser-TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/a1f39ec707a14b9e8439c25d05723795 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/a1f39ec707a14b9e8439c25d05723795 2024-11-10T03:18:44,966 DEBUG [StoreCloser-TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/f54a46f7397b4d69999ee65e5fc71e75 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/f54a46f7397b4d69999ee65e5fc71e75 2024-11-10T03:18:44,967 DEBUG [StoreCloser-TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/3e095333c68d478a9bec181daf82f0b4 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/3e095333c68d478a9bec181daf82f0b4 2024-11-10T03:18:44,968 DEBUG [StoreCloser-TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/ade3de1a1f204aa99d80a824c3add6f4 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/ade3de1a1f204aa99d80a824c3add6f4 2024-11-10T03:18:44,968 DEBUG [StoreCloser-TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/03dd4bb8490f4d92b88e175c669e18ff to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/03dd4bb8490f4d92b88e175c669e18ff 2024-11-10T03:18:44,969 DEBUG [StoreCloser-TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/94fd751b374f4f388dda0c54ee4c44c6 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/94fd751b374f4f388dda0c54ee4c44c6 2024-11-10T03:18:44,970 DEBUG [StoreCloser-TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/6f9af81163334305b75f9a10eab587b5 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/6f9af81163334305b75f9a10eab587b5 2024-11-10T03:18:44,971 DEBUG [StoreCloser-TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/9b3d89f4c56d4c5da02a3dcd995465eb to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/9b3d89f4c56d4c5da02a3dcd995465eb 2024-11-10T03:18:44,971 DEBUG [StoreCloser-TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/48b2662dda0645179f1377ff08753b36 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/48b2662dda0645179f1377ff08753b36 2024-11-10T03:18:44,972 DEBUG [StoreCloser-TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/8cce2fc8bc03498ab7089cab49637b0f to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/8cce2fc8bc03498ab7089cab49637b0f 2024-11-10T03:18:44,973 DEBUG [StoreCloser-TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/b7a3861772e04ab29fe26f7c19438a20, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/c6af7227b16448f38b1a2a17d026eafa, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/ca58c833c8594e7d952c592ed25a9b47, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/f7e331178a3b44579cb38bf46f65e95c, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/269c5d74a50e497ba998dd3ffa30ab06, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/3da789b3418b40f19c464b34ba1b10ee, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/86f143f9a349474ab7cd2fbff6e14a9d, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/320ff5ac0d7646148c237be62db37864, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/15798386f8f746039e32e36caf1414d4, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/18d4552c238544f383ea470a2031da86, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/1ece73c143124171a7b58e642b1696ed, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/259b42fcb4e545ff9feb9ded907019f5, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/9c241ab08a294a7da132353cab4c57fd, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/6090c04d37554d87a8d998e3d5eb8f3a, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/643dde3b39c44b07908de7bba5993b72, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/534ceb8e69504a26a9f7008678579b68, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/1156c1304ef044a1ac1936208d1b62f0, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/107ab47b7cdb4097812562e21cff4ec0, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/1f17baa0b11a42c59e48d2cb513e078c, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/0de5f882342b494ca95fb47d369b2d0c, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/463874149f764ee88e2b7bd88a889430, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/8261d7dd1bc44d9d8b4c00d845f83924, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/5c9b3c4d36bd476491084a0d12591821, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/5714c65d183544a5a8480ae06a41f026, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/0db7c6b1e5ae423f8e7d8fe69f7d073a, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/25d59ccf66d94e29a447cb209fdc8548, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/a55d5c8ada9c47b88e3a642820c6b727, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/6f05b943a3c442e49fc80eaf81ab92ca] to archive 2024-11-10T03:18:44,974 DEBUG [StoreCloser-TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T03:18:44,975 DEBUG [StoreCloser-TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/b7a3861772e04ab29fe26f7c19438a20 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/b7a3861772e04ab29fe26f7c19438a20 2024-11-10T03:18:44,975 DEBUG [StoreCloser-TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/c6af7227b16448f38b1a2a17d026eafa to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/c6af7227b16448f38b1a2a17d026eafa 2024-11-10T03:18:44,976 DEBUG [StoreCloser-TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/ca58c833c8594e7d952c592ed25a9b47 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/ca58c833c8594e7d952c592ed25a9b47 2024-11-10T03:18:44,976 DEBUG [StoreCloser-TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/f7e331178a3b44579cb38bf46f65e95c to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/f7e331178a3b44579cb38bf46f65e95c 2024-11-10T03:18:44,977 DEBUG [StoreCloser-TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/269c5d74a50e497ba998dd3ffa30ab06 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/269c5d74a50e497ba998dd3ffa30ab06 2024-11-10T03:18:44,978 DEBUG [StoreCloser-TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/3da789b3418b40f19c464b34ba1b10ee to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/3da789b3418b40f19c464b34ba1b10ee 2024-11-10T03:18:44,978 DEBUG [StoreCloser-TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/86f143f9a349474ab7cd2fbff6e14a9d to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/86f143f9a349474ab7cd2fbff6e14a9d 2024-11-10T03:18:44,979 DEBUG [StoreCloser-TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/320ff5ac0d7646148c237be62db37864 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/320ff5ac0d7646148c237be62db37864 2024-11-10T03:18:44,980 DEBUG [StoreCloser-TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/15798386f8f746039e32e36caf1414d4 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/15798386f8f746039e32e36caf1414d4 2024-11-10T03:18:44,980 DEBUG [StoreCloser-TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/18d4552c238544f383ea470a2031da86 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/18d4552c238544f383ea470a2031da86 2024-11-10T03:18:44,981 DEBUG [StoreCloser-TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/1ece73c143124171a7b58e642b1696ed to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/1ece73c143124171a7b58e642b1696ed 2024-11-10T03:18:44,982 DEBUG [StoreCloser-TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/259b42fcb4e545ff9feb9ded907019f5 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/259b42fcb4e545ff9feb9ded907019f5 2024-11-10T03:18:44,982 DEBUG [StoreCloser-TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/9c241ab08a294a7da132353cab4c57fd to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/9c241ab08a294a7da132353cab4c57fd 2024-11-10T03:18:44,983 DEBUG [StoreCloser-TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/6090c04d37554d87a8d998e3d5eb8f3a to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/6090c04d37554d87a8d998e3d5eb8f3a 2024-11-10T03:18:44,984 DEBUG [StoreCloser-TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/643dde3b39c44b07908de7bba5993b72 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/643dde3b39c44b07908de7bba5993b72 2024-11-10T03:18:44,984 DEBUG [StoreCloser-TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/534ceb8e69504a26a9f7008678579b68 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/534ceb8e69504a26a9f7008678579b68 2024-11-10T03:18:44,985 DEBUG [StoreCloser-TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/1156c1304ef044a1ac1936208d1b62f0 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/1156c1304ef044a1ac1936208d1b62f0 2024-11-10T03:18:44,985 DEBUG [StoreCloser-TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/107ab47b7cdb4097812562e21cff4ec0 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/107ab47b7cdb4097812562e21cff4ec0 2024-11-10T03:18:44,986 DEBUG [StoreCloser-TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/1f17baa0b11a42c59e48d2cb513e078c to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/1f17baa0b11a42c59e48d2cb513e078c 2024-11-10T03:18:44,987 DEBUG [StoreCloser-TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/0de5f882342b494ca95fb47d369b2d0c to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/0de5f882342b494ca95fb47d369b2d0c 2024-11-10T03:18:44,987 DEBUG [StoreCloser-TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/463874149f764ee88e2b7bd88a889430 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/463874149f764ee88e2b7bd88a889430 2024-11-10T03:18:44,988 DEBUG [StoreCloser-TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/8261d7dd1bc44d9d8b4c00d845f83924 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/8261d7dd1bc44d9d8b4c00d845f83924 2024-11-10T03:18:44,988 DEBUG [StoreCloser-TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/5c9b3c4d36bd476491084a0d12591821 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/5c9b3c4d36bd476491084a0d12591821 2024-11-10T03:18:44,989 DEBUG [StoreCloser-TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/5714c65d183544a5a8480ae06a41f026 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/5714c65d183544a5a8480ae06a41f026 2024-11-10T03:18:44,990 DEBUG [StoreCloser-TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/0db7c6b1e5ae423f8e7d8fe69f7d073a to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/0db7c6b1e5ae423f8e7d8fe69f7d073a 2024-11-10T03:18:44,990 DEBUG [StoreCloser-TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/25d59ccf66d94e29a447cb209fdc8548 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/25d59ccf66d94e29a447cb209fdc8548 2024-11-10T03:18:44,991 DEBUG [StoreCloser-TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/a55d5c8ada9c47b88e3a642820c6b727 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/a55d5c8ada9c47b88e3a642820c6b727 2024-11-10T03:18:44,992 DEBUG [StoreCloser-TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/6f05b943a3c442e49fc80eaf81ab92ca to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/6f05b943a3c442e49fc80eaf81ab92ca 2024-11-10T03:18:44,994 DEBUG [StoreCloser-TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/3a9c93cb787746b38064f12b2cee6994, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/fda4c38e535d4448a280e1c0fd2e809d, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/12effa3c9bd44c8c84c6d610c416c15a, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/9188e932759742d6bd067d05751fdecd, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/58b14f331f7d467ba8d7cbf299129e71, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/4c9c972d2bef45c39156fabc8e1e5ea9, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/fec6fe5cc52646b9943935d57cc89d31, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/720435b6c54c423896ed222d8a9f91bf, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/fddc2bd590ec41e193a4223a9839983d, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/1089d5af2d3f459c9341cf672a203046, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/d40e9bd982e14363be018ff5f060144c, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/bae683e3ccf045f78601495689f2d5a1, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/15213684846c4f2e9df138df025f81fb, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/42c250f31b0b4c00a6232411adc09c4f, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/4b92c491f83443e69db1f2478e4b6bf5, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/fa7b6eb5e8ed40679a5fdc72553081e8, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/969f7bb4e3bd41c4ac1602cd719985e3, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/e96e4d920ae7402e854f84322f71679a, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/c3699a88a174415dbd5c6a193317718e, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/b09350f3de7848f4b688c01ebd90ecf7, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/ae3b36192ed14b44a8ae897949fada8a, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/76d5366672b743e386814ddb90a81da1, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/e8658a2106fc48acbcb7935422e3951d, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/fe8fb2ba86dd4fdc9b7124cc21950c63, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/31e775ad811f49a3bbe8f9b56e6eada8, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/8a65cf210238477cac178fde264991c6, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/87778dacb9334a9392798a23a25c8558, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/d23f345ed3744c8eb283e7921d1db507] to archive 2024-11-10T03:18:44,995 DEBUG [StoreCloser-TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T03:18:44,996 DEBUG [StoreCloser-TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/3a9c93cb787746b38064f12b2cee6994 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/3a9c93cb787746b38064f12b2cee6994 2024-11-10T03:18:44,996 DEBUG [StoreCloser-TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/fda4c38e535d4448a280e1c0fd2e809d to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/fda4c38e535d4448a280e1c0fd2e809d 2024-11-10T03:18:44,997 DEBUG [StoreCloser-TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/12effa3c9bd44c8c84c6d610c416c15a to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/12effa3c9bd44c8c84c6d610c416c15a 2024-11-10T03:18:44,997 DEBUG [StoreCloser-TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/9188e932759742d6bd067d05751fdecd to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/9188e932759742d6bd067d05751fdecd 2024-11-10T03:18:44,998 DEBUG [StoreCloser-TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/58b14f331f7d467ba8d7cbf299129e71 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/58b14f331f7d467ba8d7cbf299129e71 2024-11-10T03:18:44,999 DEBUG [StoreCloser-TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/4c9c972d2bef45c39156fabc8e1e5ea9 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/4c9c972d2bef45c39156fabc8e1e5ea9 2024-11-10T03:18:44,999 DEBUG [StoreCloser-TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/fec6fe5cc52646b9943935d57cc89d31 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/fec6fe5cc52646b9943935d57cc89d31 2024-11-10T03:18:45,000 DEBUG [StoreCloser-TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/720435b6c54c423896ed222d8a9f91bf to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/720435b6c54c423896ed222d8a9f91bf 2024-11-10T03:18:45,001 DEBUG [StoreCloser-TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/fddc2bd590ec41e193a4223a9839983d to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/fddc2bd590ec41e193a4223a9839983d 2024-11-10T03:18:45,001 DEBUG [StoreCloser-TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/1089d5af2d3f459c9341cf672a203046 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/1089d5af2d3f459c9341cf672a203046 2024-11-10T03:18:45,002 DEBUG [StoreCloser-TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/d40e9bd982e14363be018ff5f060144c to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/d40e9bd982e14363be018ff5f060144c 2024-11-10T03:18:45,002 DEBUG [StoreCloser-TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/bae683e3ccf045f78601495689f2d5a1 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/bae683e3ccf045f78601495689f2d5a1 2024-11-10T03:18:45,003 DEBUG [StoreCloser-TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/15213684846c4f2e9df138df025f81fb to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/15213684846c4f2e9df138df025f81fb 2024-11-10T03:18:45,004 DEBUG [StoreCloser-TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/42c250f31b0b4c00a6232411adc09c4f to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/42c250f31b0b4c00a6232411adc09c4f 2024-11-10T03:18:45,005 DEBUG [StoreCloser-TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/4b92c491f83443e69db1f2478e4b6bf5 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/4b92c491f83443e69db1f2478e4b6bf5 2024-11-10T03:18:45,005 DEBUG [StoreCloser-TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/fa7b6eb5e8ed40679a5fdc72553081e8 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/fa7b6eb5e8ed40679a5fdc72553081e8 2024-11-10T03:18:45,006 DEBUG [StoreCloser-TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/969f7bb4e3bd41c4ac1602cd719985e3 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/969f7bb4e3bd41c4ac1602cd719985e3 2024-11-10T03:18:45,006 DEBUG [StoreCloser-TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/e96e4d920ae7402e854f84322f71679a to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/e96e4d920ae7402e854f84322f71679a 2024-11-10T03:18:45,007 DEBUG [StoreCloser-TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/c3699a88a174415dbd5c6a193317718e to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/c3699a88a174415dbd5c6a193317718e 2024-11-10T03:18:45,008 DEBUG [StoreCloser-TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/b09350f3de7848f4b688c01ebd90ecf7 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/b09350f3de7848f4b688c01ebd90ecf7 2024-11-10T03:18:45,008 DEBUG [StoreCloser-TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/ae3b36192ed14b44a8ae897949fada8a to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/ae3b36192ed14b44a8ae897949fada8a 2024-11-10T03:18:45,009 DEBUG [StoreCloser-TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/76d5366672b743e386814ddb90a81da1 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/76d5366672b743e386814ddb90a81da1 2024-11-10T03:18:45,010 DEBUG [StoreCloser-TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/e8658a2106fc48acbcb7935422e3951d to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/e8658a2106fc48acbcb7935422e3951d 2024-11-10T03:18:45,011 DEBUG [StoreCloser-TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/fe8fb2ba86dd4fdc9b7124cc21950c63 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/fe8fb2ba86dd4fdc9b7124cc21950c63 2024-11-10T03:18:45,011 DEBUG [StoreCloser-TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/31e775ad811f49a3bbe8f9b56e6eada8 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/31e775ad811f49a3bbe8f9b56e6eada8 2024-11-10T03:18:45,012 DEBUG [StoreCloser-TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/8a65cf210238477cac178fde264991c6 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/8a65cf210238477cac178fde264991c6 2024-11-10T03:18:45,012 DEBUG [StoreCloser-TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/87778dacb9334a9392798a23a25c8558 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/87778dacb9334a9392798a23a25c8558 2024-11-10T03:18:45,013 DEBUG [StoreCloser-TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/d23f345ed3744c8eb283e7921d1db507 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/d23f345ed3744c8eb283e7921d1db507 2024-11-10T03:18:45,016 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/recovered.edits/577.seqid, newMaxSeqId=577, maxSeqId=1 2024-11-10T03:18:45,017 INFO [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. 2024-11-10T03:18:45,017 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1676): Region close journal for 206af57796ad2b3d51883827ba0564a7: Waiting for close lock at 1731208723706Running coprocessor pre-close hooks at 1731208723706Disabling compacts and flushes for region at 1731208723706Disabling writes for close at 1731208723707 (+1 ms)Obtaining lock to block concurrent updates at 1731208723707Preparing flush snapshotting stores in 206af57796ad2b3d51883827ba0564a7 at 1731208723707Finished memstore snapshotting TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731208723707Flushing stores of TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7. at 1731208723708 (+1 ms)Flushing 206af57796ad2b3d51883827ba0564a7/A: creating writer at 1731208723708Flushing 206af57796ad2b3d51883827ba0564a7/A: appending metadata at 1731208723710 (+2 ms)Flushing 206af57796ad2b3d51883827ba0564a7/A: closing flushed file at 1731208723710Flushing 206af57796ad2b3d51883827ba0564a7/B: creating writer at 1731208724117 (+407 ms)Flushing 206af57796ad2b3d51883827ba0564a7/B: appending metadata at 1731208724120 (+3 ms)Flushing 206af57796ad2b3d51883827ba0564a7/B: closing flushed file at 1731208724120Flushing 206af57796ad2b3d51883827ba0564a7/C: creating writer at 1731208724527 (+407 ms)Flushing 206af57796ad2b3d51883827ba0564a7/C: appending metadata at 1731208724533 (+6 ms)Flushing 206af57796ad2b3d51883827ba0564a7/C: closing flushed file at 1731208724533Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@f55363a: reopening flushed file at 1731208724941 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3c08d887: reopening flushed file at 1731208724944 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1de9e06d: reopening flushed file at 1731208724946 (+2 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 206af57796ad2b3d51883827ba0564a7 in 1242ms, sequenceid=574, compaction requested=true at 1731208724949 (+3 ms)Writing region close event to WAL at 1731208725014 (+65 ms)Running coprocessor post-close hooks at 1731208725017 (+3 ms)Closed at 1731208725017 2024-11-10T03:18:45,018 INFO [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(157): Closed 206af57796ad2b3d51883827ba0564a7 2024-11-10T03:18:45,019 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=206af57796ad2b3d51883827ba0564a7, regionState=CLOSED 2024-11-10T03:18:45,020 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=144, ppid=143, state=RUNNABLE, hasLock=false; CloseRegionProcedure 206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 because future has completed 2024-11-10T03:18:45,022 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-11-10T03:18:45,022 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; CloseRegionProcedure 206af57796ad2b3d51883827ba0564a7, server=394d2222f724,38139,1731208572054 in 1.4670 sec 2024-11-10T03:18:45,023 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=142 2024-11-10T03:18:45,023 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=142, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=206af57796ad2b3d51883827ba0564a7, UNASSIGN in 1.4700 sec 2024-11-10T03:18:45,024 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-10T03:18:45,024 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4730 sec 2024-11-10T03:18:45,025 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731208725025"}]},"ts":"1731208725025"} 2024-11-10T03:18:45,026 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-10T03:18:45,026 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-10T03:18:45,027 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.4790 sec 2024-11-10T03:18:45,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-10T03:18:45,693 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-10T03:18:45,694 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-10T03:18:45,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] procedure2.ProcedureExecutor(1139): Stored pid=145, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T03:18:45,695 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=145, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T03:18:45,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-10T03:18:45,696 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=145, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T03:18:45,698 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7 2024-11-10T03:18:45,699 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A, FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B, FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C, FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/recovered.edits] 2024-11-10T03:18:45,700 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/16a241386396466dab94bc9eb5f8b782 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/16a241386396466dab94bc9eb5f8b782 2024-11-10T03:18:45,701 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/273ce4ce912a43bfa7c415fc1db3c2de to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/273ce4ce912a43bfa7c415fc1db3c2de 2024-11-10T03:18:45,702 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/787b3eb5ccb64766b27875d6d5ccbba6 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/787b3eb5ccb64766b27875d6d5ccbba6 2024-11-10T03:18:45,703 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/fab9c9223eed4c9cbb76424fbcb4d10b to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/A/fab9c9223eed4c9cbb76424fbcb4d10b 2024-11-10T03:18:45,705 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/15c4db04306e4c558050dc870111aceb to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/15c4db04306e4c558050dc870111aceb 2024-11-10T03:18:45,706 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/5d7349365190482385981bb8b0fc0e27 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/5d7349365190482385981bb8b0fc0e27 2024-11-10T03:18:45,706 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/d9c0855f5eb044eda08db2005a400998 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/d9c0855f5eb044eda08db2005a400998 2024-11-10T03:18:45,707 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/f085536494b44a0697657857547148d8 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/B/f085536494b44a0697657857547148d8 2024-11-10T03:18:45,709 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/368156794de94d3f890ec7263eb64d61 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/368156794de94d3f890ec7263eb64d61 2024-11-10T03:18:45,710 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/69b6067e89144899ad656c3d24c0e509 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/69b6067e89144899ad656c3d24c0e509 2024-11-10T03:18:45,711 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/6f4fe489ebb84341b1a1ac08bbc10452 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/6f4fe489ebb84341b1a1ac08bbc10452 2024-11-10T03:18:45,711 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/fd1ab996003241ca92f52b63667b2dd5 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/C/fd1ab996003241ca92f52b63667b2dd5 2024-11-10T03:18:45,713 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/recovered.edits/577.seqid to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7/recovered.edits/577.seqid 2024-11-10T03:18:45,713 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/206af57796ad2b3d51883827ba0564a7 2024-11-10T03:18:45,713 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-10T03:18:45,715 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=145, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T03:18:45,716 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-10T03:18:45,717 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-10T03:18:45,718 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=145, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T03:18:45,718 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-10T03:18:45,718 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731208725718"}]},"ts":"9223372036854775807"} 2024-11-10T03:18:45,720 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-10T03:18:45,720 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 206af57796ad2b3d51883827ba0564a7, NAME => 'TestAcidGuarantees,,1731208692132.206af57796ad2b3d51883827ba0564a7.', STARTKEY => '', ENDKEY => ''}] 2024-11-10T03:18:45,720 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-10T03:18:45,720 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731208725720"}]},"ts":"9223372036854775807"} 2024-11-10T03:18:45,721 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-10T03:18:45,721 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=145, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T03:18:45,722 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 27 msec 2024-11-10T03:18:45,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-10T03:18:45,803 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-10T03:18:45,803 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-10T03:18:45,812 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=246 (was 247), OpenFileDescriptor=443 (was 451), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=683 (was 710), ProcessCount=11 (was 11), AvailableMemoryMB=1346 (was 2111) 2024-11-10T03:18:45,821 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=246, OpenFileDescriptor=443, MaxFileDescriptor=1048576, SystemLoadAverage=683, ProcessCount=11, AvailableMemoryMB=1345 2024-11-10T03:18:45,822 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-10T03:18:45,822 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T03:18:45,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] procedure2.ProcedureExecutor(1139): Stored pid=146, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-10T03:18:45,824 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-10T03:18:45,824 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:18:45,824 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 146 2024-11-10T03:18:45,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-10T03:18:45,825 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-10T03:18:45,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742377_1553 (size=1061) 2024-11-10T03:18:45,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-10T03:18:46,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-10T03:18:46,235 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8 2024-11-10T03:18:46,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742378_1554 (size=53) 2024-11-10T03:18:46,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-10T03:18:46,640 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T03:18:46,640 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 07b306bf509fe2706c5b13e060d37e6b, disabling compactions & flushes 2024-11-10T03:18:46,640 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:18:46,640 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:18:46,640 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. after waiting 0 ms 2024-11-10T03:18:46,640 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:18:46,640 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:18:46,640 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 07b306bf509fe2706c5b13e060d37e6b: Waiting for close lock at 1731208726640Disabling compacts and flushes for region at 1731208726640Disabling writes for close at 1731208726640Writing region close event to WAL at 1731208726640Closed at 1731208726640 2024-11-10T03:18:46,641 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-10T03:18:46,641 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731208726641"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731208726641"}]},"ts":"1731208726641"} 2024-11-10T03:18:46,642 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-10T03:18:46,643 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-10T03:18:46,643 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731208726643"}]},"ts":"1731208726643"} 2024-11-10T03:18:46,644 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-10T03:18:46,644 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=07b306bf509fe2706c5b13e060d37e6b, ASSIGN}] 2024-11-10T03:18:46,645 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=07b306bf509fe2706c5b13e060d37e6b, ASSIGN 2024-11-10T03:18:46,646 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=07b306bf509fe2706c5b13e060d37e6b, ASSIGN; state=OFFLINE, location=394d2222f724,38139,1731208572054; forceNewPlan=false, retain=false 2024-11-10T03:18:46,796 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=07b306bf509fe2706c5b13e060d37e6b, regionState=OPENING, regionLocation=394d2222f724,38139,1731208572054 2024-11-10T03:18:46,798 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=07b306bf509fe2706c5b13e060d37e6b, ASSIGN because future has completed 2024-11-10T03:18:46,798 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=148, ppid=147, state=RUNNABLE, hasLock=false; OpenRegionProcedure 07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054}] 2024-11-10T03:18:46,952 INFO [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:18:46,952 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(7752): Opening region: {ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''} 2024-11-10T03:18:46,953 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 07b306bf509fe2706c5b13e060d37e6b 2024-11-10T03:18:46,953 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T03:18:46,953 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(7794): checking encryption for 07b306bf509fe2706c5b13e060d37e6b 2024-11-10T03:18:46,953 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(7797): checking classloading for 07b306bf509fe2706c5b13e060d37e6b 2024-11-10T03:18:46,954 INFO [StoreOpener-07b306bf509fe2706c5b13e060d37e6b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 07b306bf509fe2706c5b13e060d37e6b 2024-11-10T03:18:46,955 INFO [StoreOpener-07b306bf509fe2706c5b13e060d37e6b-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T03:18:46,956 INFO [StoreOpener-07b306bf509fe2706c5b13e060d37e6b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 07b306bf509fe2706c5b13e060d37e6b columnFamilyName A 2024-11-10T03:18:46,956 DEBUG [StoreOpener-07b306bf509fe2706c5b13e060d37e6b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:18:46,956 INFO [StoreOpener-07b306bf509fe2706c5b13e060d37e6b-1 {}] regionserver.HStore(327): Store=07b306bf509fe2706c5b13e060d37e6b/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T03:18:46,956 INFO [StoreOpener-07b306bf509fe2706c5b13e060d37e6b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 07b306bf509fe2706c5b13e060d37e6b 2024-11-10T03:18:46,957 INFO [StoreOpener-07b306bf509fe2706c5b13e060d37e6b-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T03:18:46,957 INFO [StoreOpener-07b306bf509fe2706c5b13e060d37e6b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 07b306bf509fe2706c5b13e060d37e6b columnFamilyName B 2024-11-10T03:18:46,957 DEBUG [StoreOpener-07b306bf509fe2706c5b13e060d37e6b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:18:46,957 INFO [StoreOpener-07b306bf509fe2706c5b13e060d37e6b-1 {}] regionserver.HStore(327): Store=07b306bf509fe2706c5b13e060d37e6b/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T03:18:46,958 INFO [StoreOpener-07b306bf509fe2706c5b13e060d37e6b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 07b306bf509fe2706c5b13e060d37e6b 2024-11-10T03:18:46,958 INFO [StoreOpener-07b306bf509fe2706c5b13e060d37e6b-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T03:18:46,958 INFO [StoreOpener-07b306bf509fe2706c5b13e060d37e6b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 07b306bf509fe2706c5b13e060d37e6b columnFamilyName C 2024-11-10T03:18:46,958 DEBUG [StoreOpener-07b306bf509fe2706c5b13e060d37e6b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:18:46,959 INFO [StoreOpener-07b306bf509fe2706c5b13e060d37e6b-1 {}] regionserver.HStore(327): Store=07b306bf509fe2706c5b13e060d37e6b/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T03:18:46,959 INFO [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:18:46,959 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1038): replaying wal for 07b306bf509fe2706c5b13e060d37e6b 2024-11-10T03:18:46,960 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b 2024-11-10T03:18:46,960 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b 2024-11-10T03:18:46,961 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1048): stopping wal replay for 07b306bf509fe2706c5b13e060d37e6b 2024-11-10T03:18:46,961 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1060): Cleaning up temporary data for 07b306bf509fe2706c5b13e060d37e6b 2024-11-10T03:18:46,961 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T03:18:46,962 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1093): writing seq id for 07b306bf509fe2706c5b13e060d37e6b 2024-11-10T03:18:46,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-10T03:18:46,964 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T03:18:46,964 INFO [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1114): Opened 07b306bf509fe2706c5b13e060d37e6b; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61974794, jitterRate=-0.07650360465049744}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T03:18:46,964 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 07b306bf509fe2706c5b13e060d37e6b 2024-11-10T03:18:46,965 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1006): Region open journal for 07b306bf509fe2706c5b13e060d37e6b: Running coprocessor pre-open hook at 1731208726953Writing region info on filesystem at 1731208726953Initializing all the Stores at 1731208726954 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731208726954Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731208726954Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731208726954Cleaning up temporary data from old regions at 1731208726961 (+7 ms)Running coprocessor post-open hooks at 1731208726964 (+3 ms)Region opened successfully at 1731208726964 2024-11-10T03:18:46,965 INFO [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., pid=148, masterSystemTime=1731208726950 2024-11-10T03:18:46,967 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:18:46,967 INFO [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:18:46,967 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=07b306bf509fe2706c5b13e060d37e6b, regionState=OPEN, openSeqNum=2, regionLocation=394d2222f724,38139,1731208572054 2024-11-10T03:18:46,968 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=148, ppid=147, state=RUNNABLE, hasLock=false; OpenRegionProcedure 07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 because future has completed 2024-11-10T03:18:46,971 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=148, resume processing ppid=147 2024-11-10T03:18:46,971 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, ppid=147, state=SUCCESS, hasLock=false; OpenRegionProcedure 07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 in 171 msec 2024-11-10T03:18:46,972 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-11-10T03:18:46,972 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=07b306bf509fe2706c5b13e060d37e6b, ASSIGN in 327 msec 2024-11-10T03:18:46,973 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-10T03:18:46,973 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731208726973"}]},"ts":"1731208726973"} 2024-11-10T03:18:46,974 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-10T03:18:46,975 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-10T03:18:46,976 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1530 sec 2024-11-10T03:18:47,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-10T03:18:47,974 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-10T03:18:47,974 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d326208, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:18:47,975 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 394d2222f724,37299,-1 for getting cluster id 2024-11-10T03:18:47,975 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T03:18:47,976 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f5c6727-65f7-4659-afe8-8e9a55664c78' 2024-11-10T03:18:47,976 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T03:18:47,976 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f5c6727-65f7-4659-afe8-8e9a55664c78" 2024-11-10T03:18:47,976 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1db4f580, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:18:47,976 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [394d2222f724,37299,-1] 2024-11-10T03:18:47,976 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T03:18:47,976 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:18:47,977 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46726, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T03:18:47,977 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24f5be8e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:18:47,978 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T03:18:47,978 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=394d2222f724,38139,1731208572054, seqNum=-1] 2024-11-10T03:18:47,978 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T03:18:47,979 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55540, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T03:18:47,980 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-10T03:18:47,980 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 394d2222f724,37299,1731208571229 2024-11-10T03:18:47,980 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@cd4ed7f 2024-11-10T03:18:47,980 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-10T03:18:47,981 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46728, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-10T03:18:47,983 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-10T03:18:47,983 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T03:18:47,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] procedure2.ProcedureExecutor(1139): Stored pid=149, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-10T03:18:47,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742379_1555 (size=1097) 2024-11-10T03:18:48,393 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-10T03:18:48,393 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-10T03:18:48,395 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=150, ppid=149, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-10T03:18:48,397 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=07b306bf509fe2706c5b13e060d37e6b, REOPEN/MOVE}] 2024-11-10T03:18:48,398 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=07b306bf509fe2706c5b13e060d37e6b, REOPEN/MOVE 2024-11-10T03:18:48,398 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=151 updating hbase:meta row=07b306bf509fe2706c5b13e060d37e6b, regionState=CLOSING, regionLocation=394d2222f724,38139,1731208572054 2024-11-10T03:18:48,399 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=07b306bf509fe2706c5b13e060d37e6b, REOPEN/MOVE because future has completed 2024-11-10T03:18:48,400 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-10T03:18:48,400 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=152, ppid=151, state=RUNNABLE, hasLock=false; CloseRegionProcedure 07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054}] 2024-11-10T03:18:48,551 INFO [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] handler.UnassignRegionHandler(122): Close 07b306bf509fe2706c5b13e060d37e6b 2024-11-10T03:18:48,551 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-10T03:18:48,552 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1722): Closing 07b306bf509fe2706c5b13e060d37e6b, disabling compactions & flushes 2024-11-10T03:18:48,552 INFO [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:18:48,552 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:18:48,552 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. after waiting 0 ms 2024-11-10T03:18:48,552 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:18:48,555 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-10T03:18:48,555 INFO [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:18:48,555 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1676): Region close journal for 07b306bf509fe2706c5b13e060d37e6b: Waiting for close lock at 1731208728552Running coprocessor pre-close hooks at 1731208728552Disabling compacts and flushes for region at 1731208728552Disabling writes for close at 1731208728552Writing region close event to WAL at 1731208728552Running coprocessor post-close hooks at 1731208728555 (+3 ms)Closed at 1731208728555 2024-11-10T03:18:48,555 WARN [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegionServer(3299): Not adding moved region record: 07b306bf509fe2706c5b13e060d37e6b to self. 2024-11-10T03:18:48,557 INFO [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] handler.UnassignRegionHandler(157): Closed 07b306bf509fe2706c5b13e060d37e6b 2024-11-10T03:18:48,557 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=151 updating hbase:meta row=07b306bf509fe2706c5b13e060d37e6b, regionState=CLOSED 2024-11-10T03:18:48,558 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=152, ppid=151, state=RUNNABLE, hasLock=false; CloseRegionProcedure 07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 because future has completed 2024-11-10T03:18:48,560 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=152, resume processing ppid=151 2024-11-10T03:18:48,560 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, ppid=151, state=SUCCESS, hasLock=false; CloseRegionProcedure 07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 in 159 msec 2024-11-10T03:18:48,561 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=07b306bf509fe2706c5b13e060d37e6b, REOPEN/MOVE; state=CLOSED, location=394d2222f724,38139,1731208572054; forceNewPlan=false, retain=true 2024-11-10T03:18:48,711 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=151 updating hbase:meta row=07b306bf509fe2706c5b13e060d37e6b, regionState=OPENING, regionLocation=394d2222f724,38139,1731208572054 2024-11-10T03:18:48,713 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=07b306bf509fe2706c5b13e060d37e6b, REOPEN/MOVE because future has completed 2024-11-10T03:18:48,713 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=151, state=RUNNABLE, hasLock=false; OpenRegionProcedure 07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054}] 2024-11-10T03:18:48,867 INFO [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:18:48,867 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(7752): Opening region: {ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''} 2024-11-10T03:18:48,867 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 07b306bf509fe2706c5b13e060d37e6b 2024-11-10T03:18:48,867 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T03:18:48,867 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(7794): checking encryption for 07b306bf509fe2706c5b13e060d37e6b 2024-11-10T03:18:48,867 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(7797): checking classloading for 07b306bf509fe2706c5b13e060d37e6b 2024-11-10T03:18:48,868 INFO [StoreOpener-07b306bf509fe2706c5b13e060d37e6b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 07b306bf509fe2706c5b13e060d37e6b 2024-11-10T03:18:48,868 INFO [StoreOpener-07b306bf509fe2706c5b13e060d37e6b-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T03:18:48,869 INFO [StoreOpener-07b306bf509fe2706c5b13e060d37e6b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 07b306bf509fe2706c5b13e060d37e6b columnFamilyName A 2024-11-10T03:18:48,870 DEBUG [StoreOpener-07b306bf509fe2706c5b13e060d37e6b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:18:48,870 INFO [StoreOpener-07b306bf509fe2706c5b13e060d37e6b-1 {}] regionserver.HStore(327): Store=07b306bf509fe2706c5b13e060d37e6b/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T03:18:48,870 INFO [StoreOpener-07b306bf509fe2706c5b13e060d37e6b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 07b306bf509fe2706c5b13e060d37e6b 2024-11-10T03:18:48,870 INFO [StoreOpener-07b306bf509fe2706c5b13e060d37e6b-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T03:18:48,871 INFO [StoreOpener-07b306bf509fe2706c5b13e060d37e6b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 07b306bf509fe2706c5b13e060d37e6b columnFamilyName B 2024-11-10T03:18:48,871 DEBUG [StoreOpener-07b306bf509fe2706c5b13e060d37e6b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:18:48,871 INFO [StoreOpener-07b306bf509fe2706c5b13e060d37e6b-1 {}] regionserver.HStore(327): Store=07b306bf509fe2706c5b13e060d37e6b/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T03:18:48,871 INFO [StoreOpener-07b306bf509fe2706c5b13e060d37e6b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 07b306bf509fe2706c5b13e060d37e6b 2024-11-10T03:18:48,871 INFO [StoreOpener-07b306bf509fe2706c5b13e060d37e6b-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T03:18:48,871 INFO [StoreOpener-07b306bf509fe2706c5b13e060d37e6b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 07b306bf509fe2706c5b13e060d37e6b columnFamilyName C 2024-11-10T03:18:48,871 DEBUG [StoreOpener-07b306bf509fe2706c5b13e060d37e6b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:18:48,872 INFO [StoreOpener-07b306bf509fe2706c5b13e060d37e6b-1 {}] regionserver.HStore(327): Store=07b306bf509fe2706c5b13e060d37e6b/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T03:18:48,872 INFO [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:18:48,872 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1038): replaying wal for 07b306bf509fe2706c5b13e060d37e6b 2024-11-10T03:18:48,872 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b 2024-11-10T03:18:48,873 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b 2024-11-10T03:18:48,873 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1048): stopping wal replay for 07b306bf509fe2706c5b13e060d37e6b 2024-11-10T03:18:48,873 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1060): Cleaning up temporary data for 07b306bf509fe2706c5b13e060d37e6b 2024-11-10T03:18:48,874 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T03:18:48,875 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1093): writing seq id for 07b306bf509fe2706c5b13e060d37e6b 2024-11-10T03:18:48,875 INFO [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1114): Opened 07b306bf509fe2706c5b13e060d37e6b; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67495627, jitterRate=0.005763217806816101}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T03:18:48,875 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 07b306bf509fe2706c5b13e060d37e6b 2024-11-10T03:18:48,876 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1006): Region open journal for 07b306bf509fe2706c5b13e060d37e6b: Running coprocessor pre-open hook at 1731208728867Writing region info on filesystem at 1731208728867Initializing all the Stores at 1731208728868 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731208728868Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731208728868Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731208728868Cleaning up temporary data from old regions at 1731208728873 (+5 ms)Running coprocessor post-open hooks at 1731208728875 (+2 ms)Region opened successfully at 1731208728876 (+1 ms) 2024-11-10T03:18:48,876 INFO [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., pid=153, masterSystemTime=1731208728864 2024-11-10T03:18:48,878 DEBUG [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:18:48,878 INFO [RS_OPEN_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:18:48,878 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=151 updating hbase:meta row=07b306bf509fe2706c5b13e060d37e6b, regionState=OPEN, openSeqNum=5, regionLocation=394d2222f724,38139,1731208572054 2024-11-10T03:18:48,880 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=153, ppid=151, state=RUNNABLE, hasLock=false; OpenRegionProcedure 07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 because future has completed 2024-11-10T03:18:48,882 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=151 2024-11-10T03:18:48,882 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=151, state=SUCCESS, hasLock=false; OpenRegionProcedure 07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 in 167 msec 2024-11-10T03:18:48,883 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-10T03:18:48,883 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=07b306bf509fe2706c5b13e060d37e6b, REOPEN/MOVE in 485 msec 2024-11-10T03:18:48,885 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=150, resume processing ppid=149 2024-11-10T03:18:48,885 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, ppid=149, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 489 msec 2024-11-10T03:18:48,886 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 902 msec 2024-11-10T03:18:48,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=149 2024-11-10T03:18:48,887 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-10T03:18:48,887 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25f6cc81, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:18:48,887 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 394d2222f724,37299,-1 for getting cluster id 2024-11-10T03:18:48,888 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T03:18:48,888 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f5c6727-65f7-4659-afe8-8e9a55664c78' 2024-11-10T03:18:48,888 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T03:18:48,888 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f5c6727-65f7-4659-afe8-8e9a55664c78" 2024-11-10T03:18:48,889 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7cd4d928, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:18:48,889 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [394d2222f724,37299,-1] 2024-11-10T03:18:48,889 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T03:18:48,889 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:18:48,889 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46740, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T03:18:48,890 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6135b34b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:18:48,890 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2cb0aa17, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:18:48,890 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 394d2222f724,37299,-1 for getting cluster id 2024-11-10T03:18:48,891 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T03:18:48,891 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f5c6727-65f7-4659-afe8-8e9a55664c78' 2024-11-10T03:18:48,891 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T03:18:48,891 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f5c6727-65f7-4659-afe8-8e9a55664c78" 2024-11-10T03:18:48,891 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55fafadd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:18:48,891 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [394d2222f724,37299,-1] 2024-11-10T03:18:48,892 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T03:18:48,892 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:18:48,892 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46754, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T03:18:48,893 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bdf5eaa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:18:48,893 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7af17435, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:18:48,893 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 394d2222f724,37299,-1 for getting cluster id 2024-11-10T03:18:48,893 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T03:18:48,894 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f5c6727-65f7-4659-afe8-8e9a55664c78' 2024-11-10T03:18:48,894 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T03:18:48,894 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f5c6727-65f7-4659-afe8-8e9a55664c78" 2024-11-10T03:18:48,894 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bfa199d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:18:48,894 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [394d2222f724,37299,-1] 2024-11-10T03:18:48,894 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T03:18:48,894 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:18:48,895 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46776, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T03:18:48,895 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@447bf961, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:18:48,896 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ad77ba3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:18:48,896 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 394d2222f724,37299,-1 for getting cluster id 2024-11-10T03:18:48,896 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T03:18:48,897 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f5c6727-65f7-4659-afe8-8e9a55664c78' 2024-11-10T03:18:48,897 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T03:18:48,897 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f5c6727-65f7-4659-afe8-8e9a55664c78" 2024-11-10T03:18:48,897 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4bb9bc09, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:18:48,897 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [394d2222f724,37299,-1] 2024-11-10T03:18:48,897 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T03:18:48,897 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:18:48,898 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46794, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T03:18:48,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5604f8eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:18:48,899 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@385c264c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:18:48,899 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 394d2222f724,37299,-1 for getting cluster id 2024-11-10T03:18:48,899 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T03:18:48,900 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f5c6727-65f7-4659-afe8-8e9a55664c78' 2024-11-10T03:18:48,900 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T03:18:48,900 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f5c6727-65f7-4659-afe8-8e9a55664c78" 2024-11-10T03:18:48,900 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a61e213, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:18:48,900 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [394d2222f724,37299,-1] 2024-11-10T03:18:48,900 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T03:18:48,900 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:18:48,901 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46826, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T03:18:48,901 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4179122c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:18:48,902 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e7a0a59, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:18:48,902 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 394d2222f724,37299,-1 for getting cluster id 2024-11-10T03:18:48,902 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T03:18:48,903 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f5c6727-65f7-4659-afe8-8e9a55664c78' 2024-11-10T03:18:48,903 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T03:18:48,903 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f5c6727-65f7-4659-afe8-8e9a55664c78" 2024-11-10T03:18:48,903 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25608cc7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:18:48,903 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [394d2222f724,37299,-1] 2024-11-10T03:18:48,903 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T03:18:48,903 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:18:48,904 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46844, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T03:18:48,904 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@131d299d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:18:48,904 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5735b92e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:18:48,905 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 394d2222f724,37299,-1 for getting cluster id 2024-11-10T03:18:48,905 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T03:18:48,905 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f5c6727-65f7-4659-afe8-8e9a55664c78' 2024-11-10T03:18:48,905 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T03:18:48,905 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f5c6727-65f7-4659-afe8-8e9a55664c78" 2024-11-10T03:18:48,905 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2103660d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:18:48,905 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [394d2222f724,37299,-1] 2024-11-10T03:18:48,906 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T03:18:48,906 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:18:48,906 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46856, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T03:18:48,907 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b87916b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:18:48,907 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7388bb9b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:18:48,907 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 394d2222f724,37299,-1 for getting cluster id 2024-11-10T03:18:48,907 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T03:18:48,908 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f5c6727-65f7-4659-afe8-8e9a55664c78' 2024-11-10T03:18:48,908 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T03:18:48,908 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f5c6727-65f7-4659-afe8-8e9a55664c78" 2024-11-10T03:18:48,908 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@360ce37e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:18:48,908 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [394d2222f724,37299,-1] 2024-11-10T03:18:48,908 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T03:18:48,908 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:18:48,909 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46868, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T03:18:48,909 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4eb519e0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:18:48,910 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71d93d3f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:18:48,910 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 394d2222f724,37299,-1 for getting cluster id 2024-11-10T03:18:48,910 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T03:18:48,910 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f5c6727-65f7-4659-afe8-8e9a55664c78' 2024-11-10T03:18:48,910 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T03:18:48,910 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f5c6727-65f7-4659-afe8-8e9a55664c78" 2024-11-10T03:18:48,911 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4143e3d9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:18:48,911 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [394d2222f724,37299,-1] 2024-11-10T03:18:48,911 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T03:18:48,911 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:18:48,911 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46878, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T03:18:48,912 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@684ffabe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:18:48,912 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b8edcfd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:18:48,912 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 394d2222f724,37299,-1 for getting cluster id 2024-11-10T03:18:48,912 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T03:18:48,913 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f5c6727-65f7-4659-afe8-8e9a55664c78' 2024-11-10T03:18:48,913 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T03:18:48,913 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f5c6727-65f7-4659-afe8-8e9a55664c78" 2024-11-10T03:18:48,913 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@242c3234, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:18:48,913 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [394d2222f724,37299,-1] 2024-11-10T03:18:48,913 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T03:18:48,914 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:18:48,914 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46904, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T03:18:48,914 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28d5869, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T03:18:48,915 DEBUG [Thread-2471 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T03:18:48,915 DEBUG [Thread-2467 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T03:18:48,915 DEBUG [Thread-2473 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T03:18:48,916 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=394d2222f724,38139,1731208572054, seqNum=-1] 2024-11-10T03:18:48,916 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=394d2222f724,38139,1731208572054, seqNum=-1] 2024-11-10T03:18:48,916 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=394d2222f724,38139,1731208572054, seqNum=-1] 2024-11-10T03:18:48,916 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T03:18:48,916 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T03:18:48,916 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T03:18:48,916 DEBUG [Thread-2482 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T03:18:48,916 DEBUG [Thread-2469 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T03:18:48,916 DEBUG [Thread-2484 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T03:18:48,916 DEBUG [Thread-2486 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T03:18:48,917 DEBUG [Thread-2475 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T03:18:48,917 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=394d2222f724,38139,1731208572054, seqNum=-1] 2024-11-10T03:18:48,917 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T03:18:48,917 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=394d2222f724,38139,1731208572054, seqNum=-1] 2024-11-10T03:18:48,917 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=394d2222f724,38139,1731208572054, seqNum=-1] 2024-11-10T03:18:48,917 DEBUG [Thread-2486 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=394d2222f724,38139,1731208572054, seqNum=-1] 2024-11-10T03:18:48,917 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T03:18:48,917 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T03:18:48,917 DEBUG [Thread-2486 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T03:18:48,917 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55578, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T03:18:48,918 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55548, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T03:18:48,918 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55594, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T03:18:48,918 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55604, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T03:18:48,918 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55616, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T03:18:48,918 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55562, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T03:18:48,918 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=394d2222f724,38139,1731208572054, seqNum=-1] 2024-11-10T03:18:48,918 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55596, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T03:18:48,919 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T03:18:48,919 DEBUG [Thread-2480 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T03:18:48,919 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5] 2024-11-10T03:18:48,919 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T03:18:48,920 DEBUG [Thread-2478 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T03:18:48,920 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55626, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T03:18:48,921 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=394d2222f724,38139,1731208572054, seqNum=-1] 2024-11-10T03:18:48,921 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T03:18:48,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-11-10T03:18:48,922 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5] 2024-11-10T03:18:48,922 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5] 2024-11-10T03:18:48,922 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55640, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T03:18:48,922 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5] 2024-11-10T03:18:48,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-10T03:18:48,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5] 2024-11-10T03:18:48,923 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=394d2222f724,38139,1731208572054, seqNum=-1] 2024-11-10T03:18:48,923 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T03:18:48,924 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5] 2024-11-10T03:18:48,924 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5] 2024-11-10T03:18:48,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5] 2024-11-10T03:18:48,924 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5] 2024-11-10T03:18:48,924 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T03:18:48,925 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T03:18:48,925 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T03:18:48,926 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55654, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T03:18:48,927 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5] 2024-11-10T03:18:48,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on 07b306bf509fe2706c5b13e060d37e6b 2024-11-10T03:18:48,930 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 07b306bf509fe2706c5b13e060d37e6b 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-10T03:18:48,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07b306bf509fe2706c5b13e060d37e6b, store=A 2024-11-10T03:18:48,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:18:48,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07b306bf509fe2706c5b13e060d37e6b, store=B 2024-11-10T03:18:48,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:18:48,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07b306bf509fe2706c5b13e060d37e6b, store=C 2024-11-10T03:18:48,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:18:48,950 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:48,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55626 deadline: 1731208788945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:48,951 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:48,951 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:48,951 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:48,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:48,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55616 deadline: 1731208788946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:48,951 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:48,951 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:48,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:48,951 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:48,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55562 deadline: 1731208788947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:48,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:48,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55578 deadline: 1731208788948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:48,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:48,952 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:48,952 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:48,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:48,952 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:48,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:48,954 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110756e0cb317254f63b2324e88e8cb2e6f_07b306bf509fe2706c5b13e060d37e6b is 50, key is test_row_0/A:col10/1731208728929/Put/seqid=0 2024-11-10T03:18:48,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:48,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55548 deadline: 1731208788949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:48,955 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:48,955 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:48,955 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:48,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742380_1556 (size=17034) 2024-11-10T03:18:48,965 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:18:48,968 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110756e0cb317254f63b2324e88e8cb2e6f_07b306bf509fe2706c5b13e060d37e6b to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110756e0cb317254f63b2324e88e8cb2e6f_07b306bf509fe2706c5b13e060d37e6b 2024-11-10T03:18:48,969 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/A/b1d23fd2280d4ae8a723eb890c968779, store: [table=TestAcidGuarantees family=A region=07b306bf509fe2706c5b13e060d37e6b] 2024-11-10T03:18:48,969 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/A/b1d23fd2280d4ae8a723eb890c968779 is 175, key is test_row_0/A:col10/1731208728929/Put/seqid=0 2024-11-10T03:18:48,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742381_1557 (size=48139) 2024-11-10T03:18:49,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-10T03:18:49,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:49,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55626 deadline: 1731208789052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:49,055 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:49,055 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:49,055 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:49,055 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:49,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55616 deadline: 1731208789053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:49,055 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:49,055 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:49,055 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:49,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:49,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55578 deadline: 1731208789063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:49,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:49,065 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:49,066 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:49,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55562 deadline: 1731208789063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:49,066 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:49,066 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:49,066 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:49,066 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:49,066 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:49,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55548 deadline: 1731208789063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:49,066 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:49,066 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:49,066 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:49,076 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-10T03:18:49,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:18:49,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. as already flushing 2024-11-10T03:18:49,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:18:49,077 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:49,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:49,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:49,229 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-10T03:18:49,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:18:49,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. as already flushing 2024-11-10T03:18:49,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:18:49,230 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:49,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:49,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:49,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-10T03:18:49,267 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:49,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55626 deadline: 1731208789263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:49,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:49,267 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:49,267 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:49,267 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:49,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55616 deadline: 1731208789263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:49,268 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:49,268 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:49,268 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:49,275 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:49,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55578 deadline: 1731208789272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:49,275 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:49,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55562 deadline: 1731208789273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:49,276 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:49,276 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:49,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55548 deadline: 1731208789273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:49,276 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:49,276 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:49,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:49,276 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:49,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:49,276 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:49,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:49,276 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:49,382 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-10T03:18:49,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:18:49,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. as already flushing 2024-11-10T03:18:49,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:18:49,382 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:49,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:49,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:49,392 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/A/b1d23fd2280d4ae8a723eb890c968779 2024-11-10T03:18:49,409 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/B/34beb169150f42499b6fb0520d48d360 is 50, key is test_row_0/B:col10/1731208728929/Put/seqid=0 2024-11-10T03:18:49,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742382_1558 (size=12001) 2024-11-10T03:18:49,535 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-10T03:18:49,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:18:49,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. as already flushing 2024-11-10T03:18:49,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:18:49,535 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:49,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:49,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:49,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-10T03:18:49,576 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:49,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55626 deadline: 1731208789573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:49,576 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:49,576 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:49,576 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:49,576 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:49,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55616 deadline: 1731208789573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:49,576 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:49,576 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:49,576 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:49,583 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:49,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55578 deadline: 1731208789582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:49,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:49,584 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:49,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:49,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:49,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55548 deadline: 1731208789583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:49,584 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:49,584 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:49,584 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:49,584 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:49,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55562 deadline: 1731208789583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:49,584 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:49,584 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:49,584 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:49,692 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-10T03:18:49,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:18:49,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. as already flushing 2024-11-10T03:18:49,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:18:49,696 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:49,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:49,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:49,813 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/B/34beb169150f42499b6fb0520d48d360 2024-11-10T03:18:49,836 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/C/883ceba6a39a4c32879c19f0efbfe667 is 50, key is test_row_0/C:col10/1731208728929/Put/seqid=0 2024-11-10T03:18:49,848 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-10T03:18:49,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:18:49,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. as already flushing 2024-11-10T03:18:49,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:18:49,849 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:49,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:49,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:49,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742383_1559 (size=12001) 2024-11-10T03:18:49,852 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/C/883ceba6a39a4c32879c19f0efbfe667 2024-11-10T03:18:49,856 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/A/b1d23fd2280d4ae8a723eb890c968779 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/b1d23fd2280d4ae8a723eb890c968779 2024-11-10T03:18:49,862 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/b1d23fd2280d4ae8a723eb890c968779, entries=250, sequenceid=16, filesize=47.0 K 2024-11-10T03:18:49,863 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/B/34beb169150f42499b6fb0520d48d360 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/34beb169150f42499b6fb0520d48d360 2024-11-10T03:18:49,867 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/34beb169150f42499b6fb0520d48d360, entries=150, sequenceid=16, filesize=11.7 K 2024-11-10T03:18:49,867 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/C/883ceba6a39a4c32879c19f0efbfe667 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/883ceba6a39a4c32879c19f0efbfe667 2024-11-10T03:18:49,871 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/883ceba6a39a4c32879c19f0efbfe667, entries=150, sequenceid=16, filesize=11.7 K 2024-11-10T03:18:49,872 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 07b306bf509fe2706c5b13e060d37e6b in 942ms, sequenceid=16, compaction requested=false 2024-11-10T03:18:49,872 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 07b306bf509fe2706c5b13e060d37e6b: 2024-11-10T03:18:50,000 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-10T03:18:50,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:18:50,001 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing 07b306bf509fe2706c5b13e060d37e6b 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-10T03:18:50,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07b306bf509fe2706c5b13e060d37e6b, store=A 2024-11-10T03:18:50,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:18:50,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07b306bf509fe2706c5b13e060d37e6b, store=B 2024-11-10T03:18:50,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:18:50,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07b306bf509fe2706c5b13e060d37e6b, store=C 2024-11-10T03:18:50,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:18:50,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110c0a3337544a64bdf8d4f4522e483acf2_07b306bf509fe2706c5b13e060d37e6b is 50, key is test_row_0/A:col10/1731208728946/Put/seqid=0 2024-11-10T03:18:50,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742384_1560 (size=12154) 2024-11-10T03:18:50,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:18:50,046 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110c0a3337544a64bdf8d4f4522e483acf2_07b306bf509fe2706c5b13e060d37e6b to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110c0a3337544a64bdf8d4f4522e483acf2_07b306bf509fe2706c5b13e060d37e6b 2024-11-10T03:18:50,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/A/0750e0f42c38424b83cbe7d376492cf3, store: [table=TestAcidGuarantees family=A region=07b306bf509fe2706c5b13e060d37e6b] 2024-11-10T03:18:50,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/A/0750e0f42c38424b83cbe7d376492cf3 is 175, key is test_row_0/A:col10/1731208728946/Put/seqid=0 2024-11-10T03:18:50,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742385_1561 (size=30955) 2024-11-10T03:18:50,059 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=36, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/A/0750e0f42c38424b83cbe7d376492cf3 2024-11-10T03:18:50,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-10T03:18:50,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/B/7cad940719884d5fb0e686c473e0ab5f is 50, key is test_row_0/B:col10/1731208728946/Put/seqid=0 2024-11-10T03:18:50,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742386_1562 (size=12001) 2024-11-10T03:18:50,069 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/B/7cad940719884d5fb0e686c473e0ab5f 2024-11-10T03:18:50,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/C/c77c98d7cc2d498f9b2ffa442c14daed is 50, key is test_row_0/C:col10/1731208728946/Put/seqid=0 2024-11-10T03:18:50,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on 07b306bf509fe2706c5b13e060d37e6b 2024-11-10T03:18:50,085 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. as already flushing 2024-11-10T03:18:50,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742387_1563 (size=12001) 2024-11-10T03:18:50,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:50,133 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:50,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55562 deadline: 1731208790127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:50,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55548 deadline: 1731208790127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:50,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:50,134 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:50,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:50,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:50,134 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:50,134 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:50,137 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:50,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55626 deadline: 1731208790132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:50,137 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:50,137 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:50,137 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:50,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:50,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55616 deadline: 1731208790134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:50,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:50,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55578 deadline: 1731208790134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:50,138 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:50,138 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:50,138 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:50,138 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:50,138 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:50,138 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:50,246 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:50,246 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:50,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55548 deadline: 1731208790243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:50,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55562 deadline: 1731208790243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:50,246 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:50,246 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:50,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:50,246 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:50,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55626 deadline: 1731208790243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:50,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:50,246 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:50,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:50,246 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:50,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55616 deadline: 1731208790243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:50,246 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:50,246 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:50,246 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:50,246 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:50,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55578 deadline: 1731208790243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:50,247 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:50,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:50,247 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:50,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:50,247 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:50,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:50,454 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:50,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55562 deadline: 1731208790453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:50,455 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:50,455 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:50,455 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:50,455 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:50,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55616 deadline: 1731208790453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:50,455 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:50,455 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:50,455 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:50,455 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:50,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55626 deadline: 1731208790453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:50,455 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:50,455 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:50,455 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:50,455 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:50,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55578 deadline: 1731208790454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:50,456 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:50,456 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:50,456 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:50,458 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:50,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55548 deadline: 1731208790454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:50,458 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:50,458 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:50,458 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:50,494 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/C/c77c98d7cc2d498f9b2ffa442c14daed 2024-11-10T03:18:50,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/A/0750e0f42c38424b83cbe7d376492cf3 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/0750e0f42c38424b83cbe7d376492cf3 2024-11-10T03:18:50,500 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/0750e0f42c38424b83cbe7d376492cf3, entries=150, sequenceid=36, filesize=30.2 K 2024-11-10T03:18:50,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/B/7cad940719884d5fb0e686c473e0ab5f as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/7cad940719884d5fb0e686c473e0ab5f 2024-11-10T03:18:50,509 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/7cad940719884d5fb0e686c473e0ab5f, entries=150, sequenceid=36, filesize=11.7 K 2024-11-10T03:18:50,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/C/c77c98d7cc2d498f9b2ffa442c14daed as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/c77c98d7cc2d498f9b2ffa442c14daed 2024-11-10T03:18:50,512 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/c77c98d7cc2d498f9b2ffa442c14daed, entries=150, sequenceid=36, filesize=11.7 K 2024-11-10T03:18:50,513 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 07b306bf509fe2706c5b13e060d37e6b in 512ms, sequenceid=36, compaction requested=false 2024-11-10T03:18:50,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for 07b306bf509fe2706c5b13e060d37e6b: 2024-11-10T03:18:50,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:18:50,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-11-10T03:18:50,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-11-10T03:18:50,515 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-10T03:18:50,515 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5890 sec 2024-11-10T03:18:50,517 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 1.5970 sec 2024-11-10T03:18:50,549 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-10T03:18:50,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on 07b306bf509fe2706c5b13e060d37e6b 2024-11-10T03:18:50,765 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 07b306bf509fe2706c5b13e060d37e6b 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-10T03:18:50,765 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07b306bf509fe2706c5b13e060d37e6b, store=A 2024-11-10T03:18:50,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:18:50,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07b306bf509fe2706c5b13e060d37e6b, store=B 2024-11-10T03:18:50,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:18:50,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07b306bf509fe2706c5b13e060d37e6b, store=C 2024-11-10T03:18:50,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:18:50,778 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110177c790c5d2d4600bfed7a71ce546835_07b306bf509fe2706c5b13e060d37e6b is 50, key is test_row_0/A:col10/1731208730131/Put/seqid=0 2024-11-10T03:18:50,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742388_1564 (size=14594) 2024-11-10T03:18:50,782 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:18:50,785 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110177c790c5d2d4600bfed7a71ce546835_07b306bf509fe2706c5b13e060d37e6b to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110177c790c5d2d4600bfed7a71ce546835_07b306bf509fe2706c5b13e060d37e6b 2024-11-10T03:18:50,785 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/A/5b33ff44395d4a16b2e28d68edf8d8c3, store: [table=TestAcidGuarantees family=A region=07b306bf509fe2706c5b13e060d37e6b] 2024-11-10T03:18:50,786 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/A/5b33ff44395d4a16b2e28d68edf8d8c3 is 175, key is test_row_0/A:col10/1731208730131/Put/seqid=0 2024-11-10T03:18:50,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:50,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55616 deadline: 1731208790782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:50,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:50,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55578 deadline: 1731208790783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:50,789 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:50,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742389_1565 (size=39549) 2024-11-10T03:18:50,790 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:50,790 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:50,790 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:50,790 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:50,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:50,790 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:50,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55562 deadline: 1731208790783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:50,790 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=50, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/A/5b33ff44395d4a16b2e28d68edf8d8c3 2024-11-10T03:18:50,790 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:50,790 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:50,790 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:50,793 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:50,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55626 deadline: 1731208790788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:50,794 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:50,794 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:50,794 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:50,795 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:50,795 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/B/2aa06ccc9b244d3a925ab45f8b2fdff1 is 50, key is test_row_0/B:col10/1731208730131/Put/seqid=0 2024-11-10T03:18:50,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55548 deadline: 1731208790789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:50,796 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:50,796 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:50,796 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:50,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742390_1566 (size=12001) 2024-11-10T03:18:50,894 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:50,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55616 deadline: 1731208790893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:50,895 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:50,895 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:50,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55578 deadline: 1731208790893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:50,895 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:50,895 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:50,895 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:50,895 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:50,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55562 deadline: 1731208790893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:50,895 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:50,895 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:50,895 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:50,895 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:50,895 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:50,905 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:50,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55626 deadline: 1731208790903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:50,905 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:50,905 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:50,905 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:50,905 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:50,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55548 deadline: 1731208790903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:50,905 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:50,905 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:50,905 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:51,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-10T03:18:51,073 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T03:18:51,073 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T03:18:51,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-10T03:18:51,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-10T03:18:51,075 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T03:18:51,076 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T03:18:51,076 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T03:18:51,106 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:51,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55562 deadline: 1731208791103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:51,107 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:51,107 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:51,107 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:51,107 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:51,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55578 deadline: 1731208791104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:51,107 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:51,107 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:51,107 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:51,107 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:51,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55616 deadline: 1731208791104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:51,108 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:51,108 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:51,108 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:51,115 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:51,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55626 deadline: 1731208791113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:51,115 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:51,115 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:51,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55548 deadline: 1731208791113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:51,115 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:51,115 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:51,115 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:51,115 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:51,115 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:51,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-10T03:18:51,213 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/B/2aa06ccc9b244d3a925ab45f8b2fdff1 2024-11-10T03:18:51,220 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/C/5b7b606fd1444ddca74f08e77fe91496 is 50, key is test_row_0/C:col10/1731208730131/Put/seqid=0 2024-11-10T03:18:51,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742391_1567 (size=12001) 2024-11-10T03:18:51,227 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-10T03:18:51,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:18:51,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. as already flushing 2024-11-10T03:18:51,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:18:51,228 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:51,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:51,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:51,380 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-10T03:18:51,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:18:51,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. as already flushing 2024-11-10T03:18:51,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:18:51,381 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:51,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:51,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:51,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-10T03:18:51,416 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:51,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55562 deadline: 1731208791413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:51,417 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:51,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55578 deadline: 1731208791413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:51,417 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:51,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55616 deadline: 1731208791413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:51,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:51,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:51,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:51,418 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:51,418 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:51,418 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:51,418 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:51,418 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:51,418 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:51,427 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:51,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55548 deadline: 1731208791424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:51,427 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:51,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55626 deadline: 1731208791424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:51,427 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:51,428 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:51,428 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:51,428 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:51,428 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:51,428 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:51,533 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-10T03:18:51,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:18:51,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. as already flushing 2024-11-10T03:18:51,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:18:51,534 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:51,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:51,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:51,623 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/C/5b7b606fd1444ddca74f08e77fe91496 2024-11-10T03:18:51,627 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/A/5b33ff44395d4a16b2e28d68edf8d8c3 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/5b33ff44395d4a16b2e28d68edf8d8c3 2024-11-10T03:18:51,630 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/5b33ff44395d4a16b2e28d68edf8d8c3, entries=200, sequenceid=50, filesize=38.6 K 2024-11-10T03:18:51,631 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/B/2aa06ccc9b244d3a925ab45f8b2fdff1 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/2aa06ccc9b244d3a925ab45f8b2fdff1 2024-11-10T03:18:51,634 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/2aa06ccc9b244d3a925ab45f8b2fdff1, entries=150, sequenceid=50, filesize=11.7 K 2024-11-10T03:18:51,636 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/C/5b7b606fd1444ddca74f08e77fe91496 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/5b7b606fd1444ddca74f08e77fe91496 2024-11-10T03:18:51,639 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/5b7b606fd1444ddca74f08e77fe91496, entries=150, sequenceid=50, filesize=11.7 K 2024-11-10T03:18:51,640 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for 07b306bf509fe2706c5b13e060d37e6b in 875ms, sequenceid=50, compaction requested=true 2024-11-10T03:18:51,640 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 07b306bf509fe2706c5b13e060d37e6b: 2024-11-10T03:18:51,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 07b306bf509fe2706c5b13e060d37e6b:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T03:18:51,640 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T03:18:51,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:18:51,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 07b306bf509fe2706c5b13e060d37e6b:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T03:18:51,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:18:51,640 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T03:18:51,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 07b306bf509fe2706c5b13e060d37e6b:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T03:18:51,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T03:18:51,641 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T03:18:51,641 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1541): 07b306bf509fe2706c5b13e060d37e6b/B is initiating minor compaction (all files) 2024-11-10T03:18:51,641 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 07b306bf509fe2706c5b13e060d37e6b/B in TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:18:51,641 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/34beb169150f42499b6fb0520d48d360, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/7cad940719884d5fb0e686c473e0ab5f, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/2aa06ccc9b244d3a925ab45f8b2fdff1] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp, totalSize=35.2 K 2024-11-10T03:18:51,641 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 118643 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T03:18:51,641 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1541): 07b306bf509fe2706c5b13e060d37e6b/A is initiating minor compaction (all files) 2024-11-10T03:18:51,641 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 07b306bf509fe2706c5b13e060d37e6b/A in TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:18:51,641 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/b1d23fd2280d4ae8a723eb890c968779, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/0750e0f42c38424b83cbe7d376492cf3, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/5b33ff44395d4a16b2e28d68edf8d8c3] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp, totalSize=115.9 K 2024-11-10T03:18:51,641 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:18:51,641 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. files: [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/b1d23fd2280d4ae8a723eb890c968779, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/0750e0f42c38424b83cbe7d376492cf3, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/5b33ff44395d4a16b2e28d68edf8d8c3] 2024-11-10T03:18:51,642 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting b1d23fd2280d4ae8a723eb890c968779, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731208728925 2024-11-10T03:18:51,642 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 34beb169150f42499b6fb0520d48d360, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731208728928 2024-11-10T03:18:51,642 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0750e0f42c38424b83cbe7d376492cf3, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1731208728939 2024-11-10T03:18:51,642 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 7cad940719884d5fb0e686c473e0ab5f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1731208728939 2024-11-10T03:18:51,642 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b33ff44395d4a16b2e28d68edf8d8c3, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731208730131 2024-11-10T03:18:51,642 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 2aa06ccc9b244d3a925ab45f8b2fdff1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731208730131 2024-11-10T03:18:51,648 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=07b306bf509fe2706c5b13e060d37e6b] 2024-11-10T03:18:51,649 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 07b306bf509fe2706c5b13e060d37e6b#B#compaction#480 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:18:51,649 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/B/6083849e7b91410bbbabfda04b979196 is 50, key is test_row_0/B:col10/1731208730131/Put/seqid=0 2024-11-10T03:18:51,661 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111000ca5ccbe50c46e182a9f49ce5eca332_07b306bf509fe2706c5b13e060d37e6b store=[table=TestAcidGuarantees family=A region=07b306bf509fe2706c5b13e060d37e6b] 2024-11-10T03:18:51,662 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111000ca5ccbe50c46e182a9f49ce5eca332_07b306bf509fe2706c5b13e060d37e6b, store=[table=TestAcidGuarantees family=A region=07b306bf509fe2706c5b13e060d37e6b] 2024-11-10T03:18:51,662 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111000ca5ccbe50c46e182a9f49ce5eca332_07b306bf509fe2706c5b13e060d37e6b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=07b306bf509fe2706c5b13e060d37e6b] 2024-11-10T03:18:51,686 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-10T03:18:51,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:18:51,686 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing 07b306bf509fe2706c5b13e060d37e6b 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-10T03:18:51,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07b306bf509fe2706c5b13e060d37e6b, store=A 2024-11-10T03:18:51,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:18:51,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07b306bf509fe2706c5b13e060d37e6b, store=B 2024-11-10T03:18:51,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:18:51,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07b306bf509fe2706c5b13e060d37e6b, store=C 2024-11-10T03:18:51,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:18:51,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742392_1568 (size=12104) 2024-11-10T03:18:51,699 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/B/6083849e7b91410bbbabfda04b979196 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/6083849e7b91410bbbabfda04b979196 2024-11-10T03:18:51,702 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 07b306bf509fe2706c5b13e060d37e6b/B of 07b306bf509fe2706c5b13e060d37e6b into 6083849e7b91410bbbabfda04b979196(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:18:51,702 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 07b306bf509fe2706c5b13e060d37e6b: 2024-11-10T03:18:51,702 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., storeName=07b306bf509fe2706c5b13e060d37e6b/B, priority=13, startTime=1731208731640; duration=0sec 2024-11-10T03:18:51,702 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T03:18:51,702 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 07b306bf509fe2706c5b13e060d37e6b:B 2024-11-10T03:18:51,702 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T03:18:51,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-10T03:18:51,704 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T03:18:51,704 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1541): 07b306bf509fe2706c5b13e060d37e6b/C is initiating minor compaction (all files) 2024-11-10T03:18:51,704 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 07b306bf509fe2706c5b13e060d37e6b/C in TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:18:51,704 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/883ceba6a39a4c32879c19f0efbfe667, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/c77c98d7cc2d498f9b2ffa442c14daed, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/5b7b606fd1444ddca74f08e77fe91496] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp, totalSize=35.2 K 2024-11-10T03:18:51,704 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 883ceba6a39a4c32879c19f0efbfe667, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731208728928 2024-11-10T03:18:51,704 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting c77c98d7cc2d498f9b2ffa442c14daed, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1731208728939 2024-11-10T03:18:51,705 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b7b606fd1444ddca74f08e77fe91496, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731208730131 2024-11-10T03:18:51,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742393_1569 (size=4469) 2024-11-10T03:18:51,722 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 07b306bf509fe2706c5b13e060d37e6b#C#compaction#482 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:18:51,722 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/C/4c4aa8792d714e2bb6f8dc31d4745074 is 50, key is test_row_0/C:col10/1731208730131/Put/seqid=0 2024-11-10T03:18:51,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411103964da02837a4bd793dae4b1534b88fc_07b306bf509fe2706c5b13e060d37e6b is 50, key is test_row_0/A:col10/1731208730787/Put/seqid=0 2024-11-10T03:18:51,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742395_1571 (size=12154) 2024-11-10T03:18:51,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742394_1570 (size=12104) 2024-11-10T03:18:51,761 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/C/4c4aa8792d714e2bb6f8dc31d4745074 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/4c4aa8792d714e2bb6f8dc31d4745074 2024-11-10T03:18:51,765 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 07b306bf509fe2706c5b13e060d37e6b/C of 07b306bf509fe2706c5b13e060d37e6b into 4c4aa8792d714e2bb6f8dc31d4745074(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:18:51,765 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 07b306bf509fe2706c5b13e060d37e6b: 2024-11-10T03:18:51,765 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., storeName=07b306bf509fe2706c5b13e060d37e6b/C, priority=13, startTime=1731208731640; duration=0sec 2024-11-10T03:18:51,765 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:18:51,765 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 07b306bf509fe2706c5b13e060d37e6b:C 2024-11-10T03:18:51,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on 07b306bf509fe2706c5b13e060d37e6b 2024-11-10T03:18:51,925 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. as already flushing 2024-11-10T03:18:51,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:51,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55548 deadline: 1731208791943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:51,949 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:51,949 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:51,949 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:51,951 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:51,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55616 deadline: 1731208791947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:51,952 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:51,952 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:51,952 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:51,953 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:51,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55626 deadline: 1731208791948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:51,953 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:51,953 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:51,953 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:51,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:51,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55562 deadline: 1731208791948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:51,954 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:51,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55578 deadline: 1731208791948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:51,954 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:51,954 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:51,954 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:51,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:51,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:51,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:52,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:52,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55616 deadline: 1731208792063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:52,067 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:52,067 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:52,067 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:52,067 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:52,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55626 deadline: 1731208792063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:52,067 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:52,067 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:52,067 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:52,067 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:52,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55578 deadline: 1731208792063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:52,068 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:52,068 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:52,068 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:52,068 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:52,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55562 deadline: 1731208792063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:52,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:52,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:52,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:52,119 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 07b306bf509fe2706c5b13e060d37e6b#A#compaction#481 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:18:52,120 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/A/86c672fa6e1b452493e686dc87a872aa is 175, key is test_row_0/A:col10/1731208730131/Put/seqid=0 2024-11-10T03:18:52,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742396_1572 (size=31058) 2024-11-10T03:18:52,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:18:52,155 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411103964da02837a4bd793dae4b1534b88fc_07b306bf509fe2706c5b13e060d37e6b to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411103964da02837a4bd793dae4b1534b88fc_07b306bf509fe2706c5b13e060d37e6b 2024-11-10T03:18:52,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/A/f06bf60d296046a9b4e5418f2854966f, store: [table=TestAcidGuarantees family=A region=07b306bf509fe2706c5b13e060d37e6b] 2024-11-10T03:18:52,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/A/f06bf60d296046a9b4e5418f2854966f is 175, key is test_row_0/A:col10/1731208730787/Put/seqid=0 2024-11-10T03:18:52,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742397_1573 (size=30955) 2024-11-10T03:18:52,166 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=69, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/A/f06bf60d296046a9b4e5418f2854966f 2024-11-10T03:18:52,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/B/49ac62fb2fac4c7da76e3cec5f269287 is 50, key is test_row_0/B:col10/1731208730787/Put/seqid=0 2024-11-10T03:18:52,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742398_1574 (size=12001) 2024-11-10T03:18:52,182 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=69 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/B/49ac62fb2fac4c7da76e3cec5f269287 2024-11-10T03:18:52,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/C/6881928e13984d9bbf564f012d2f729c is 50, key is test_row_0/C:col10/1731208730787/Put/seqid=0 2024-11-10T03:18:52,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742399_1575 (size=12001) 2024-11-10T03:18:52,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-10T03:18:52,276 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:52,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55616 deadline: 1731208792273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:52,276 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:52,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55626 deadline: 1731208792273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:52,276 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:52,276 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:52,276 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:52,276 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:52,276 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:52,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55562 deadline: 1731208792273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:52,276 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:52,276 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:52,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:52,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:52,276 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:52,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:52,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55578 deadline: 1731208792273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:52,277 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:52,277 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:52,277 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:52,527 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/A/86c672fa6e1b452493e686dc87a872aa as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/86c672fa6e1b452493e686dc87a872aa 2024-11-10T03:18:52,531 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 07b306bf509fe2706c5b13e060d37e6b/A of 07b306bf509fe2706c5b13e060d37e6b into 86c672fa6e1b452493e686dc87a872aa(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:18:52,532 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 07b306bf509fe2706c5b13e060d37e6b: 2024-11-10T03:18:52,532 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., storeName=07b306bf509fe2706c5b13e060d37e6b/A, priority=13, startTime=1731208731640; duration=0sec 2024-11-10T03:18:52,532 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:18:52,532 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 07b306bf509fe2706c5b13e060d37e6b:A 2024-11-10T03:18:52,586 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:52,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55616 deadline: 1731208792583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:52,586 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:52,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55626 deadline: 1731208792583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:52,586 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:52,586 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:52,586 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:52,586 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:52,586 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:52,586 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:52,586 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:52,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55578 deadline: 1731208792583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:52,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:52,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:52,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:52,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55562 deadline: 1731208792583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:52,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:52,587 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:52,587 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:52,587 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:52,596 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=69 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/C/6881928e13984d9bbf564f012d2f729c 2024-11-10T03:18:52,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/A/f06bf60d296046a9b4e5418f2854966f as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/f06bf60d296046a9b4e5418f2854966f 2024-11-10T03:18:52,604 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/f06bf60d296046a9b4e5418f2854966f, entries=150, sequenceid=69, filesize=30.2 K 2024-11-10T03:18:52,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/B/49ac62fb2fac4c7da76e3cec5f269287 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/49ac62fb2fac4c7da76e3cec5f269287 2024-11-10T03:18:52,608 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/49ac62fb2fac4c7da76e3cec5f269287, entries=150, sequenceid=69, filesize=11.7 K 2024-11-10T03:18:52,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/C/6881928e13984d9bbf564f012d2f729c as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/6881928e13984d9bbf564f012d2f729c 2024-11-10T03:18:52,611 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/6881928e13984d9bbf564f012d2f729c, entries=150, sequenceid=69, filesize=11.7 K 2024-11-10T03:18:52,613 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=80.51 KB/82440 for 07b306bf509fe2706c5b13e060d37e6b in 926ms, sequenceid=69, compaction requested=false 2024-11-10T03:18:52,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for 07b306bf509fe2706c5b13e060d37e6b: 2024-11-10T03:18:52,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:18:52,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-10T03:18:52,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-10T03:18:52,616 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-10T03:18:52,616 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5380 sec 2024-11-10T03:18:52,617 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 1.5420 sec 2024-11-10T03:18:52,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on 07b306bf509fe2706c5b13e060d37e6b 2024-11-10T03:18:52,964 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 07b306bf509fe2706c5b13e060d37e6b 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-10T03:18:52,964 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07b306bf509fe2706c5b13e060d37e6b, store=A 2024-11-10T03:18:52,964 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:18:52,964 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07b306bf509fe2706c5b13e060d37e6b, store=B 2024-11-10T03:18:52,964 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:18:52,964 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07b306bf509fe2706c5b13e060d37e6b, store=C 2024-11-10T03:18:52,964 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:18:52,985 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111064f1c054c46c4af197975388095cb66e_07b306bf509fe2706c5b13e060d37e6b is 50, key is test_row_0/A:col10/1731208732963/Put/seqid=0 2024-11-10T03:18:52,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742400_1576 (size=12154) 2024-11-10T03:18:53,025 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:53,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55548 deadline: 1731208793022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:53,025 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:53,025 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:53,025 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:53,095 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:53,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55616 deadline: 1731208793092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:53,095 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:53,095 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:53,095 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:53,095 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:53,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55578 deadline: 1731208793092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:53,096 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:53,096 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:53,096 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:53,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55626 deadline: 1731208793092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:53,096 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:53,096 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:53,096 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:53,096 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:53,096 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:53,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55562 deadline: 1731208793093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:53,096 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:53,096 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:53,096 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:53,136 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:53,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55548 deadline: 1731208793133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:53,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:53,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:53,137 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:53,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-10T03:18:53,223 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T03:18:53,224 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T03:18:53,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-10T03:18:53,226 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T03:18:53,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-10T03:18:53,227 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T03:18:53,227 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T03:18:53,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-10T03:18:53,344 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:53,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55548 deadline: 1731208793343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:53,344 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:53,344 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:53,344 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:53,379 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-10T03:18:53,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:18:53,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. as already flushing 2024-11-10T03:18:53,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:18:53,379 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:53,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:53,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:53,390 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:18:53,393 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111064f1c054c46c4af197975388095cb66e_07b306bf509fe2706c5b13e060d37e6b to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111064f1c054c46c4af197975388095cb66e_07b306bf509fe2706c5b13e060d37e6b 2024-11-10T03:18:53,394 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/A/757c0046f47e4aa0baba7c72e9dc8762, store: [table=TestAcidGuarantees family=A region=07b306bf509fe2706c5b13e060d37e6b] 2024-11-10T03:18:53,394 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/A/757c0046f47e4aa0baba7c72e9dc8762 is 175, key is test_row_0/A:col10/1731208732963/Put/seqid=0 2024-11-10T03:18:53,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742401_1577 (size=30955) 2024-11-10T03:18:53,531 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-10T03:18:53,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:18:53,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. as already flushing 2024-11-10T03:18:53,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:18:53,532 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:53,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:53,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:53,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-10T03:18:53,655 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:53,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55548 deadline: 1731208793653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:53,655 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:53,655 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:53,655 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:53,684 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-10T03:18:53,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:18:53,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. as already flushing 2024-11-10T03:18:53,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:18:53,684 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:53,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:53,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:53,802 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=88, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/A/757c0046f47e4aa0baba7c72e9dc8762 2024-11-10T03:18:53,808 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/B/6ce48c361c694785b0c7603472751c44 is 50, key is test_row_0/B:col10/1731208732963/Put/seqid=0 2024-11-10T03:18:53,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742402_1578 (size=12001) 2024-11-10T03:18:53,819 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/B/6ce48c361c694785b0c7603472751c44 2024-11-10T03:18:53,830 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/C/d18082c3579b489ab025dc5446509982 is 50, key is test_row_0/C:col10/1731208732963/Put/seqid=0 2024-11-10T03:18:53,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742403_1579 (size=12001) 2024-11-10T03:18:53,836 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-10T03:18:53,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:18:53,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. as already flushing 2024-11-10T03:18:53,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:18:53,837 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:53,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:53,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:53,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-10T03:18:53,989 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-10T03:18:53,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:18:53,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. as already flushing 2024-11-10T03:18:53,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:18:53,989 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:53,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:53,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:54,105 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:54,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55616 deadline: 1731208794103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:54,105 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:54,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55626 deadline: 1731208794103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:54,106 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 394d2222f724,38139,1731208572054 for 'test_row_2' in 07b306bf509fe2706c5b13e060d37e6b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2158 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T03:18:54,106 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:54,106 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:54,106 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:54,106 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 394d2222f724,38139,1731208572054 for 'test_row_2' in 07b306bf509fe2706c5b13e060d37e6b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2158 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T03:18:54,106 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:54,106 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:54,106 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:54,117 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:54,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55562 deadline: 1731208794113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:54,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:54,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55578 deadline: 1731208794113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:54,118 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 394d2222f724,38139,1731208572054 for 'test_row_2' in 07b306bf509fe2706c5b13e060d37e6b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T03:18:54,118 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:54,118 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:54,118 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:54,118 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 394d2222f724,38139,1731208572054 for 'test_row_0' in 07b306bf509fe2706c5b13e060d37e6b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T03:18:54,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:54,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:54,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:54,142 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-10T03:18:54,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:18:54,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. as already flushing 2024-11-10T03:18:54,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:18:54,142 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:54,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:54,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:54,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:54,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55548 deadline: 1731208794163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:54,165 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:54,165 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:54,165 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:54,236 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/C/d18082c3579b489ab025dc5446509982 2024-11-10T03:18:54,240 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/A/757c0046f47e4aa0baba7c72e9dc8762 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/757c0046f47e4aa0baba7c72e9dc8762 2024-11-10T03:18:54,244 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/757c0046f47e4aa0baba7c72e9dc8762, entries=150, sequenceid=88, filesize=30.2 K 2024-11-10T03:18:54,245 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/B/6ce48c361c694785b0c7603472751c44 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/6ce48c361c694785b0c7603472751c44 2024-11-10T03:18:54,248 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/6ce48c361c694785b0c7603472751c44, entries=150, sequenceid=88, filesize=11.7 K 2024-11-10T03:18:54,248 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/C/d18082c3579b489ab025dc5446509982 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/d18082c3579b489ab025dc5446509982 2024-11-10T03:18:54,251 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/d18082c3579b489ab025dc5446509982, entries=150, sequenceid=88, filesize=11.7 K 2024-11-10T03:18:54,252 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 07b306bf509fe2706c5b13e060d37e6b in 1288ms, sequenceid=88, compaction requested=true 2024-11-10T03:18:54,252 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 07b306bf509fe2706c5b13e060d37e6b: 2024-11-10T03:18:54,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 07b306bf509fe2706c5b13e060d37e6b:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T03:18:54,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:18:54,252 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T03:18:54,252 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T03:18:54,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 07b306bf509fe2706c5b13e060d37e6b:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T03:18:54,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:18:54,253 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92968 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T03:18:54,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 07b306bf509fe2706c5b13e060d37e6b:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T03:18:54,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T03:18:54,253 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1541): 07b306bf509fe2706c5b13e060d37e6b/A is initiating minor compaction (all files) 2024-11-10T03:18:54,253 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 07b306bf509fe2706c5b13e060d37e6b/A in TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:18:54,254 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/86c672fa6e1b452493e686dc87a872aa, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/f06bf60d296046a9b4e5418f2854966f, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/757c0046f47e4aa0baba7c72e9dc8762] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp, totalSize=90.8 K 2024-11-10T03:18:54,254 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:18:54,254 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. files: [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/86c672fa6e1b452493e686dc87a872aa, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/f06bf60d296046a9b4e5418f2854966f, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/757c0046f47e4aa0baba7c72e9dc8762] 2024-11-10T03:18:54,254 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T03:18:54,254 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1541): 07b306bf509fe2706c5b13e060d37e6b/B is initiating minor compaction (all files) 2024-11-10T03:18:54,254 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 07b306bf509fe2706c5b13e060d37e6b/B in TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:18:54,254 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 86c672fa6e1b452493e686dc87a872aa, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731208730131 2024-11-10T03:18:54,254 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/6083849e7b91410bbbabfda04b979196, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/49ac62fb2fac4c7da76e3cec5f269287, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/6ce48c361c694785b0c7603472751c44] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp, totalSize=35.3 K 2024-11-10T03:18:54,254 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting f06bf60d296046a9b4e5418f2854966f, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=69, earliestPutTs=1731208730781 2024-11-10T03:18:54,254 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 6083849e7b91410bbbabfda04b979196, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731208730131 2024-11-10T03:18:54,254 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 757c0046f47e4aa0baba7c72e9dc8762, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1731208731939 2024-11-10T03:18:54,254 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 49ac62fb2fac4c7da76e3cec5f269287, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=69, earliestPutTs=1731208730781 2024-11-10T03:18:54,255 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ce48c361c694785b0c7603472751c44, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1731208731939 2024-11-10T03:18:54,266 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=07b306bf509fe2706c5b13e060d37e6b] 2024-11-10T03:18:54,267 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 07b306bf509fe2706c5b13e060d37e6b#B#compaction#489 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:18:54,268 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/B/6a48249176dd453bb6ac913226d066e2 is 50, key is test_row_0/B:col10/1731208732963/Put/seqid=0 2024-11-10T03:18:54,268 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111078ad36d8d06e4f5c83999802e4f33dc1_07b306bf509fe2706c5b13e060d37e6b store=[table=TestAcidGuarantees family=A region=07b306bf509fe2706c5b13e060d37e6b] 2024-11-10T03:18:54,270 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111078ad36d8d06e4f5c83999802e4f33dc1_07b306bf509fe2706c5b13e060d37e6b, store=[table=TestAcidGuarantees family=A region=07b306bf509fe2706c5b13e060d37e6b] 2024-11-10T03:18:54,270 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111078ad36d8d06e4f5c83999802e4f33dc1_07b306bf509fe2706c5b13e060d37e6b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=07b306bf509fe2706c5b13e060d37e6b] 2024-11-10T03:18:54,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742405_1581 (size=4469) 2024-11-10T03:18:54,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742404_1580 (size=12207) 2024-11-10T03:18:54,275 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 07b306bf509fe2706c5b13e060d37e6b#A#compaction#490 average throughput is 2.71 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:18:54,276 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/A/a723af900541489fbdd25c3e8fe6675f is 175, key is test_row_0/A:col10/1731208732963/Put/seqid=0 2024-11-10T03:18:54,279 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/B/6a48249176dd453bb6ac913226d066e2 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/6a48249176dd453bb6ac913226d066e2 2024-11-10T03:18:54,284 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 07b306bf509fe2706c5b13e060d37e6b/B of 07b306bf509fe2706c5b13e060d37e6b into 6a48249176dd453bb6ac913226d066e2(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:18:54,284 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 07b306bf509fe2706c5b13e060d37e6b: 2024-11-10T03:18:54,284 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., storeName=07b306bf509fe2706c5b13e060d37e6b/B, priority=13, startTime=1731208734252; duration=0sec 2024-11-10T03:18:54,284 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T03:18:54,284 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 07b306bf509fe2706c5b13e060d37e6b:B 2024-11-10T03:18:54,284 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T03:18:54,285 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T03:18:54,285 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1541): 07b306bf509fe2706c5b13e060d37e6b/C is initiating minor compaction (all files) 2024-11-10T03:18:54,285 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 07b306bf509fe2706c5b13e060d37e6b/C in TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:18:54,285 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/4c4aa8792d714e2bb6f8dc31d4745074, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/6881928e13984d9bbf564f012d2f729c, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/d18082c3579b489ab025dc5446509982] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp, totalSize=35.3 K 2024-11-10T03:18:54,285 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c4aa8792d714e2bb6f8dc31d4745074, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731208730131 2024-11-10T03:18:54,286 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 6881928e13984d9bbf564f012d2f729c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=69, earliestPutTs=1731208730781 2024-11-10T03:18:54,286 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting d18082c3579b489ab025dc5446509982, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1731208731939 2024-11-10T03:18:54,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742406_1582 (size=31161) 2024-11-10T03:18:54,294 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/A/a723af900541489fbdd25c3e8fe6675f as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/a723af900541489fbdd25c3e8fe6675f 2024-11-10T03:18:54,295 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-10T03:18:54,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:18:54,295 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing 07b306bf509fe2706c5b13e060d37e6b 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-10T03:18:54,295 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 07b306bf509fe2706c5b13e060d37e6b#C#compaction#491 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:18:54,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07b306bf509fe2706c5b13e060d37e6b, store=A 2024-11-10T03:18:54,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:18:54,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07b306bf509fe2706c5b13e060d37e6b, store=B 2024-11-10T03:18:54,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:18:54,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07b306bf509fe2706c5b13e060d37e6b, store=C 2024-11-10T03:18:54,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:18:54,296 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/C/cd75bf348ab94f5ab0b4bf8d15d4770c is 50, key is test_row_0/C:col10/1731208732963/Put/seqid=0 2024-11-10T03:18:54,301 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 07b306bf509fe2706c5b13e060d37e6b/A of 07b306bf509fe2706c5b13e060d37e6b into a723af900541489fbdd25c3e8fe6675f(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:18:54,301 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 07b306bf509fe2706c5b13e060d37e6b: 2024-11-10T03:18:54,301 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., storeName=07b306bf509fe2706c5b13e060d37e6b/A, priority=13, startTime=1731208734252; duration=0sec 2024-11-10T03:18:54,301 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:18:54,302 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 07b306bf509fe2706c5b13e060d37e6b:A 2024-11-10T03:18:54,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742407_1583 (size=12207) 2024-11-10T03:18:54,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411100416aea5e8b94445987c1d13c1a4cf0b_07b306bf509fe2706c5b13e060d37e6b is 50, key is test_row_0/A:col10/1731208732997/Put/seqid=0 2024-11-10T03:18:54,337 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/C/cd75bf348ab94f5ab0b4bf8d15d4770c as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/cd75bf348ab94f5ab0b4bf8d15d4770c 2024-11-10T03:18:54,341 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 07b306bf509fe2706c5b13e060d37e6b/C of 07b306bf509fe2706c5b13e060d37e6b into cd75bf348ab94f5ab0b4bf8d15d4770c(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:18:54,342 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 07b306bf509fe2706c5b13e060d37e6b: 2024-11-10T03:18:54,342 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., storeName=07b306bf509fe2706c5b13e060d37e6b/C, priority=13, startTime=1731208734253; duration=0sec 2024-11-10T03:18:54,342 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:18:54,342 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 07b306bf509fe2706c5b13e060d37e6b:C 2024-11-10T03:18:54,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742408_1584 (size=12154) 2024-11-10T03:18:54,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-10T03:18:54,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:18:54,749 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411100416aea5e8b94445987c1d13c1a4cf0b_07b306bf509fe2706c5b13e060d37e6b to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411100416aea5e8b94445987c1d13c1a4cf0b_07b306bf509fe2706c5b13e060d37e6b 2024-11-10T03:18:54,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/A/eb197861ff5744c8a1b86e27afe577de, store: [table=TestAcidGuarantees family=A region=07b306bf509fe2706c5b13e060d37e6b] 2024-11-10T03:18:54,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/A/eb197861ff5744c8a1b86e27afe577de is 175, key is test_row_0/A:col10/1731208732997/Put/seqid=0 2024-11-10T03:18:54,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742409_1585 (size=30955) 2024-11-10T03:18:55,186 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=105, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/A/eb197861ff5744c8a1b86e27afe577de 2024-11-10T03:18:55,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on 07b306bf509fe2706c5b13e060d37e6b 2024-11-10T03:18:55,188 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. as already flushing 2024-11-10T03:18:55,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/B/1710ac918c814b55a9cedf78c4f49d69 is 50, key is test_row_0/B:col10/1731208732997/Put/seqid=0 2024-11-10T03:18:55,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742410_1586 (size=12001) 2024-11-10T03:18:55,261 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:55,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55548 deadline: 1731208795259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:55,261 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:55,261 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:55,261 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:55,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-10T03:18:55,374 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:55,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55548 deadline: 1731208795373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:55,374 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:55,374 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:55,374 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:55,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:55,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55548 deadline: 1731208795582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:55,584 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:55,584 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:55,584 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:55,605 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/B/1710ac918c814b55a9cedf78c4f49d69 2024-11-10T03:18:55,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/C/923469d6fc70494b90c3fb4b9e791348 is 50, key is test_row_0/C:col10/1731208732997/Put/seqid=0 2024-11-10T03:18:55,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742411_1587 (size=12001) 2024-11-10T03:18:55,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:55,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55548 deadline: 1731208795893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:55,896 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:55,896 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:55,896 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:56,014 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/C/923469d6fc70494b90c3fb4b9e791348 2024-11-10T03:18:56,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/A/eb197861ff5744c8a1b86e27afe577de as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/eb197861ff5744c8a1b86e27afe577de 2024-11-10T03:18:56,021 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/eb197861ff5744c8a1b86e27afe577de, entries=150, sequenceid=105, filesize=30.2 K 2024-11-10T03:18:56,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/B/1710ac918c814b55a9cedf78c4f49d69 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/1710ac918c814b55a9cedf78c4f49d69 2024-11-10T03:18:56,024 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/1710ac918c814b55a9cedf78c4f49d69, entries=150, sequenceid=105, filesize=11.7 K 2024-11-10T03:18:56,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/C/923469d6fc70494b90c3fb4b9e791348 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/923469d6fc70494b90c3fb4b9e791348 2024-11-10T03:18:56,027 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/923469d6fc70494b90c3fb4b9e791348, entries=150, sequenceid=105, filesize=11.7 K 2024-11-10T03:18:56,028 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 07b306bf509fe2706c5b13e060d37e6b in 1733ms, sequenceid=105, compaction requested=false 2024-11-10T03:18:56,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for 07b306bf509fe2706c5b13e060d37e6b: 2024-11-10T03:18:56,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:18:56,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-10T03:18:56,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-10T03:18:56,032 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-10T03:18:56,032 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8020 sec 2024-11-10T03:18:56,037 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 2.8110 sec 2024-11-10T03:18:56,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on 07b306bf509fe2706c5b13e060d37e6b 2024-11-10T03:18:56,126 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 07b306bf509fe2706c5b13e060d37e6b 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-10T03:18:56,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07b306bf509fe2706c5b13e060d37e6b, store=A 2024-11-10T03:18:56,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:18:56,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07b306bf509fe2706c5b13e060d37e6b, store=B 2024-11-10T03:18:56,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:18:56,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07b306bf509fe2706c5b13e060d37e6b, store=C 2024-11-10T03:18:56,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:18:56,141 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111084849330c3a646f7bac56f00b4fa8f9c_07b306bf509fe2706c5b13e060d37e6b is 50, key is test_row_0/A:col10/1731208735257/Put/seqid=0 2024-11-10T03:18:56,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742412_1588 (size=14594) 2024-11-10T03:18:56,159 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:56,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55562 deadline: 1731208796158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:56,159 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:56,159 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:56,159 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:56,160 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:56,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55616 deadline: 1731208796159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:56,160 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:56,160 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:56,160 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:56,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:56,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55626 deadline: 1731208796163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:56,164 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:56,164 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:56,164 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:56,173 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:56,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55578 deadline: 1731208796172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:56,173 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:56,173 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:56,173 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:56,264 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:56,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55562 deadline: 1731208796263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:56,264 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:56,264 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:56,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55616 deadline: 1731208796263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:56,264 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:56,265 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:56,265 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:56,265 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:56,265 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:56,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:56,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55626 deadline: 1731208796272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:56,275 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:56,275 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:56,275 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:56,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:56,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55578 deadline: 1731208796283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:56,286 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:56,286 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:56,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:56,405 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:56,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55548 deadline: 1731208796402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:56,405 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:56,405 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:56,405 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:56,475 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:56,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55562 deadline: 1731208796473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:56,476 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:56,476 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:56,476 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:56,476 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:56,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55616 deadline: 1731208796473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:56,476 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:56,476 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:56,476 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:56,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:56,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55626 deadline: 1731208796482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:56,484 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:56,484 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:56,484 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:56,496 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:56,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55578 deadline: 1731208796492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:56,496 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:56,496 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:56,497 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:56,556 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:18:56,567 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111084849330c3a646f7bac56f00b4fa8f9c_07b306bf509fe2706c5b13e060d37e6b to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111084849330c3a646f7bac56f00b4fa8f9c_07b306bf509fe2706c5b13e060d37e6b 2024-11-10T03:18:56,568 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/A/f256e8b29d5c43148f01d7fc82b35581, store: [table=TestAcidGuarantees family=A region=07b306bf509fe2706c5b13e060d37e6b] 2024-11-10T03:18:56,568 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/A/f256e8b29d5c43148f01d7fc82b35581 is 175, key is test_row_0/A:col10/1731208735257/Put/seqid=0 2024-11-10T03:18:56,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742413_1589 (size=39549) 2024-11-10T03:18:56,785 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:56,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55562 deadline: 1731208796783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:56,786 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:56,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:56,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55616 deadline: 1731208796783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:56,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:56,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:56,786 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:56,786 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:56,786 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:56,795 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:56,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55626 deadline: 1731208796794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:56,796 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:56,796 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:56,796 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:56,806 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:56,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55578 deadline: 1731208796803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:56,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:56,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:56,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:56,972 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=124, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/A/f256e8b29d5c43148f01d7fc82b35581 2024-11-10T03:18:56,979 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/B/1517f097ce6746a5bdbd03f3e535717c is 50, key is test_row_0/B:col10/1731208735257/Put/seqid=0 2024-11-10T03:18:56,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742414_1590 (size=12001) 2024-11-10T03:18:57,002 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=124 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/B/1517f097ce6746a5bdbd03f3e535717c 2024-11-10T03:18:57,011 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/C/914a8ec541274fcc9b8f212bc1c61d8c is 50, key is test_row_0/C:col10/1731208735257/Put/seqid=0 2024-11-10T03:18:57,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742415_1591 (size=12001) 2024-11-10T03:18:57,032 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=124 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/C/914a8ec541274fcc9b8f212bc1c61d8c 2024-11-10T03:18:57,035 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/A/f256e8b29d5c43148f01d7fc82b35581 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/f256e8b29d5c43148f01d7fc82b35581 2024-11-10T03:18:57,040 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/f256e8b29d5c43148f01d7fc82b35581, entries=200, sequenceid=124, filesize=38.6 K 2024-11-10T03:18:57,040 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/B/1517f097ce6746a5bdbd03f3e535717c as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/1517f097ce6746a5bdbd03f3e535717c 2024-11-10T03:18:57,047 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/1517f097ce6746a5bdbd03f3e535717c, entries=150, sequenceid=124, filesize=11.7 K 2024-11-10T03:18:57,048 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/C/914a8ec541274fcc9b8f212bc1c61d8c as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/914a8ec541274fcc9b8f212bc1c61d8c 2024-11-10T03:18:57,051 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/914a8ec541274fcc9b8f212bc1c61d8c, entries=150, sequenceid=124, filesize=11.7 K 2024-11-10T03:18:57,052 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 07b306bf509fe2706c5b13e060d37e6b in 926ms, sequenceid=124, compaction requested=true 2024-11-10T03:18:57,052 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 07b306bf509fe2706c5b13e060d37e6b: 2024-11-10T03:18:57,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 07b306bf509fe2706c5b13e060d37e6b:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T03:18:57,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:18:57,052 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T03:18:57,052 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T03:18:57,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 07b306bf509fe2706c5b13e060d37e6b:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T03:18:57,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:18:57,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 07b306bf509fe2706c5b13e060d37e6b:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T03:18:57,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T03:18:57,053 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T03:18:57,053 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1541): 07b306bf509fe2706c5b13e060d37e6b/B is initiating minor compaction (all files) 2024-11-10T03:18:57,053 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 07b306bf509fe2706c5b13e060d37e6b/B in TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:18:57,053 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/6a48249176dd453bb6ac913226d066e2, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/1710ac918c814b55a9cedf78c4f49d69, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/1517f097ce6746a5bdbd03f3e535717c] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp, totalSize=35.4 K 2024-11-10T03:18:57,054 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101665 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T03:18:57,054 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1541): 07b306bf509fe2706c5b13e060d37e6b/A is initiating minor compaction (all files) 2024-11-10T03:18:57,054 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 07b306bf509fe2706c5b13e060d37e6b/A in TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:18:57,054 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/a723af900541489fbdd25c3e8fe6675f, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/eb197861ff5744c8a1b86e27afe577de, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/f256e8b29d5c43148f01d7fc82b35581] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp, totalSize=99.3 K 2024-11-10T03:18:57,054 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:18:57,054 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. files: [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/a723af900541489fbdd25c3e8fe6675f, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/eb197861ff5744c8a1b86e27afe577de, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/f256e8b29d5c43148f01d7fc82b35581] 2024-11-10T03:18:57,055 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a48249176dd453bb6ac913226d066e2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1731208731939 2024-11-10T03:18:57,055 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting a723af900541489fbdd25c3e8fe6675f, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1731208731939 2024-11-10T03:18:57,056 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 1710ac918c814b55a9cedf78c4f49d69, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1731208732997 2024-11-10T03:18:57,056 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting eb197861ff5744c8a1b86e27afe577de, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1731208732997 2024-11-10T03:18:57,056 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 1517f097ce6746a5bdbd03f3e535717c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=124, earliestPutTs=1731208735247 2024-11-10T03:18:57,056 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting f256e8b29d5c43148f01d7fc82b35581, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=124, earliestPutTs=1731208735247 2024-11-10T03:18:57,081 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=07b306bf509fe2706c5b13e060d37e6b] 2024-11-10T03:18:57,086 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 07b306bf509fe2706c5b13e060d37e6b#B#compaction#499 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:18:57,086 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/B/fafa0676e1944bb5a4886af093d6f596 is 50, key is test_row_0/B:col10/1731208735257/Put/seqid=0 2024-11-10T03:18:57,096 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111034109f3561864d79a7cf8290ab56f2f3_07b306bf509fe2706c5b13e060d37e6b store=[table=TestAcidGuarantees family=A region=07b306bf509fe2706c5b13e060d37e6b] 2024-11-10T03:18:57,099 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111034109f3561864d79a7cf8290ab56f2f3_07b306bf509fe2706c5b13e060d37e6b, store=[table=TestAcidGuarantees family=A region=07b306bf509fe2706c5b13e060d37e6b] 2024-11-10T03:18:57,099 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111034109f3561864d79a7cf8290ab56f2f3_07b306bf509fe2706c5b13e060d37e6b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=07b306bf509fe2706c5b13e060d37e6b] 2024-11-10T03:18:57,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742416_1592 (size=12309) 2024-11-10T03:18:57,143 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/B/fafa0676e1944bb5a4886af093d6f596 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/fafa0676e1944bb5a4886af093d6f596 2024-11-10T03:18:57,148 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 07b306bf509fe2706c5b13e060d37e6b/B of 07b306bf509fe2706c5b13e060d37e6b into fafa0676e1944bb5a4886af093d6f596(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:18:57,148 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 07b306bf509fe2706c5b13e060d37e6b: 2024-11-10T03:18:57,148 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., storeName=07b306bf509fe2706c5b13e060d37e6b/B, priority=13, startTime=1731208737052; duration=0sec 2024-11-10T03:18:57,148 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T03:18:57,148 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 07b306bf509fe2706c5b13e060d37e6b:B 2024-11-10T03:18:57,148 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T03:18:57,149 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T03:18:57,149 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1541): 07b306bf509fe2706c5b13e060d37e6b/C is initiating minor compaction (all files) 2024-11-10T03:18:57,149 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 07b306bf509fe2706c5b13e060d37e6b/C in TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:18:57,149 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/cd75bf348ab94f5ab0b4bf8d15d4770c, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/923469d6fc70494b90c3fb4b9e791348, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/914a8ec541274fcc9b8f212bc1c61d8c] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp, totalSize=35.4 K 2024-11-10T03:18:57,150 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting cd75bf348ab94f5ab0b4bf8d15d4770c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1731208731939 2024-11-10T03:18:57,150 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 923469d6fc70494b90c3fb4b9e791348, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1731208732997 2024-11-10T03:18:57,150 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 914a8ec541274fcc9b8f212bc1c61d8c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=124, earliestPutTs=1731208735247 2024-11-10T03:18:57,163 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 07b306bf509fe2706c5b13e060d37e6b#C#compaction#500 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:18:57,163 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/C/d4e9e04dafeb4e63ace5b0879c745b2c is 50, key is test_row_0/C:col10/1731208735257/Put/seqid=0 2024-11-10T03:18:57,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742417_1593 (size=4469) 2024-11-10T03:18:57,166 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 07b306bf509fe2706c5b13e060d37e6b#A#compaction#498 average throughput is 0.29 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:18:57,167 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/A/685288664520438eb094b998765b5264 is 175, key is test_row_0/A:col10/1731208735257/Put/seqid=0 2024-11-10T03:18:57,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742418_1594 (size=12309) 2024-11-10T03:18:57,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742419_1595 (size=31263) 2024-11-10T03:18:57,180 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/C/d4e9e04dafeb4e63ace5b0879c745b2c as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/d4e9e04dafeb4e63ace5b0879c745b2c 2024-11-10T03:18:57,185 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/A/685288664520438eb094b998765b5264 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/685288664520438eb094b998765b5264 2024-11-10T03:18:57,187 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 07b306bf509fe2706c5b13e060d37e6b/C of 07b306bf509fe2706c5b13e060d37e6b into d4e9e04dafeb4e63ace5b0879c745b2c(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:18:57,187 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 07b306bf509fe2706c5b13e060d37e6b: 2024-11-10T03:18:57,188 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., storeName=07b306bf509fe2706c5b13e060d37e6b/C, priority=13, startTime=1731208737053; duration=0sec 2024-11-10T03:18:57,188 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:18:57,188 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 07b306bf509fe2706c5b13e060d37e6b:C 2024-11-10T03:18:57,193 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 07b306bf509fe2706c5b13e060d37e6b/A of 07b306bf509fe2706c5b13e060d37e6b into 685288664520438eb094b998765b5264(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:18:57,193 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 07b306bf509fe2706c5b13e060d37e6b: 2024-11-10T03:18:57,193 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., storeName=07b306bf509fe2706c5b13e060d37e6b/A, priority=13, startTime=1731208737052; duration=0sec 2024-11-10T03:18:57,193 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:18:57,193 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 07b306bf509fe2706c5b13e060d37e6b:A 2024-11-10T03:18:57,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on 07b306bf509fe2706c5b13e060d37e6b 2024-11-10T03:18:57,295 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 07b306bf509fe2706c5b13e060d37e6b 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-10T03:18:57,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07b306bf509fe2706c5b13e060d37e6b, store=A 2024-11-10T03:18:57,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:18:57,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07b306bf509fe2706c5b13e060d37e6b, store=B 2024-11-10T03:18:57,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:18:57,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07b306bf509fe2706c5b13e060d37e6b, store=C 2024-11-10T03:18:57,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:18:57,302 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111049c8d7284f7748d9b421309a1ebca726_07b306bf509fe2706c5b13e060d37e6b is 50, key is test_row_0/A:col10/1731208736156/Put/seqid=0 2024-11-10T03:18:57,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742420_1596 (size=17284) 2024-11-10T03:18:57,318 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:57,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55616 deadline: 1731208797314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:57,318 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:57,318 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:57,318 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:57,322 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:57,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55626 deadline: 1731208797317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:57,322 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:57,322 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:57,322 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:57,322 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:57,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55562 deadline: 1731208797318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:57,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:57,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:57,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:57,323 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:57,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55578 deadline: 1731208797318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:57,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:57,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:57,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:57,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-10T03:18:57,393 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T03:18:57,394 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T03:18:57,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-11-10T03:18:57,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-10T03:18:57,396 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T03:18:57,397 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T03:18:57,397 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T03:18:57,424 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:57,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55548 deadline: 1731208797422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:57,424 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:57,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55616 deadline: 1731208797422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:57,424 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 394d2222f724,38139,1731208572054 for 'test_row_2' in 07b306bf509fe2706c5b13e060d37e6b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T03:18:57,425 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:57,425 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:57,425 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:57,425 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:57,425 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:57,425 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:57,434 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:57,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55626 deadline: 1731208797433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:57,435 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:57,435 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:57,435 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:57,435 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:57,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55562 deadline: 1731208797433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:57,435 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:57,435 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:57,435 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:57,435 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:57,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55578 deadline: 1731208797433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:57,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:57,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:57,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:57,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-10T03:18:57,549 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-10T03:18:57,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:18:57,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. as already flushing 2024-11-10T03:18:57,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:18:57,549 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:57,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:57,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:57,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:57,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55616 deadline: 1731208797633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:57,636 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:57,636 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:57,636 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:57,646 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:57,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55626 deadline: 1731208797643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:57,647 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:57,647 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:57,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55562 deadline: 1731208797643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:57,647 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:57,647 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:57,647 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:57,647 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:57,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55578 deadline: 1731208797644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:57,647 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:57,647 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:57,647 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:57,647 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:57,648 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:57,709 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:18:57,713 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-10T03:18:57,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:18:57,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. as already flushing 2024-11-10T03:18:57,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:18:57,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-10T03:18:57,713 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:57,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:57,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:57,723 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111049c8d7284f7748d9b421309a1ebca726_07b306bf509fe2706c5b13e060d37e6b to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111049c8d7284f7748d9b421309a1ebca726_07b306bf509fe2706c5b13e060d37e6b 2024-11-10T03:18:57,725 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/A/e9322848bab7494cbc8d7e60323f53ba, store: [table=TestAcidGuarantees family=A region=07b306bf509fe2706c5b13e060d37e6b] 2024-11-10T03:18:57,725 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/A/e9322848bab7494cbc8d7e60323f53ba is 175, key is test_row_0/A:col10/1731208736156/Put/seqid=0 2024-11-10T03:18:57,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742421_1597 (size=48389) 2024-11-10T03:18:57,730 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=144, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/A/e9322848bab7494cbc8d7e60323f53ba 2024-11-10T03:18:57,739 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/B/49a3aa1e4c82453f856b12109cc877dd is 50, key is test_row_0/B:col10/1731208736156/Put/seqid=0 2024-11-10T03:18:57,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742422_1598 (size=12151) 2024-11-10T03:18:57,780 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=144 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/B/49a3aa1e4c82453f856b12109cc877dd 2024-11-10T03:18:57,793 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/C/5a2116a0c4364a12b011d795caf31789 is 50, key is test_row_0/C:col10/1731208736156/Put/seqid=0 2024-11-10T03:18:57,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742423_1599 (size=12151) 2024-11-10T03:18:57,802 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=144 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/C/5a2116a0c4364a12b011d795caf31789 2024-11-10T03:18:57,805 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/A/e9322848bab7494cbc8d7e60323f53ba as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/e9322848bab7494cbc8d7e60323f53ba 2024-11-10T03:18:57,809 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/e9322848bab7494cbc8d7e60323f53ba, entries=250, sequenceid=144, filesize=47.3 K 2024-11-10T03:18:57,810 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/B/49a3aa1e4c82453f856b12109cc877dd as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/49a3aa1e4c82453f856b12109cc877dd 2024-11-10T03:18:57,814 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/49a3aa1e4c82453f856b12109cc877dd, entries=150, sequenceid=144, filesize=11.9 K 2024-11-10T03:18:57,815 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/C/5a2116a0c4364a12b011d795caf31789 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/5a2116a0c4364a12b011d795caf31789 2024-11-10T03:18:57,817 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/5a2116a0c4364a12b011d795caf31789, entries=150, sequenceid=144, filesize=11.9 K 2024-11-10T03:18:57,818 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for 07b306bf509fe2706c5b13e060d37e6b in 523ms, sequenceid=144, compaction requested=false 2024-11-10T03:18:57,818 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 07b306bf509fe2706c5b13e060d37e6b: 2024-11-10T03:18:57,866 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-10T03:18:57,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:18:57,866 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing 07b306bf509fe2706c5b13e060d37e6b 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-10T03:18:57,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07b306bf509fe2706c5b13e060d37e6b, store=A 2024-11-10T03:18:57,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:18:57,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07b306bf509fe2706c5b13e060d37e6b, store=B 2024-11-10T03:18:57,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:18:57,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07b306bf509fe2706c5b13e060d37e6b, store=C 2024-11-10T03:18:57,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:18:57,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411105b2cd91846c04c348ffc2d0b9d4bee6c_07b306bf509fe2706c5b13e060d37e6b is 50, key is test_row_0/A:col10/1731208737317/Put/seqid=0 2024-11-10T03:18:57,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742424_1600 (size=12304) 2024-11-10T03:18:57,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on 07b306bf509fe2706c5b13e060d37e6b 2024-11-10T03:18:57,946 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. as already flushing 2024-11-10T03:18:57,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:57,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55562 deadline: 1731208797966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:57,972 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:57,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55616 deadline: 1731208797968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:57,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:57,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:57,973 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:57,973 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:57,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:57,973 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:57,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55626 deadline: 1731208797968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:57,973 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:57,973 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:57,973 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:57,973 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:57,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:57,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55578 deadline: 1731208797970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:57,975 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:57,975 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:57,975 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:58,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-10T03:18:58,084 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:58,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55616 deadline: 1731208798083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:58,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:58,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55578 deadline: 1731208798083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:58,085 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:58,085 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:58,085 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:58,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:58,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:58,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:58,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:58,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55562 deadline: 1731208798083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:58,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:58,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:58,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:58,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:58,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55626 deadline: 1731208798083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:58,085 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:58,086 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:58,086 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:58,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:18:58,283 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411105b2cd91846c04c348ffc2d0b9d4bee6c_07b306bf509fe2706c5b13e060d37e6b to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411105b2cd91846c04c348ffc2d0b9d4bee6c_07b306bf509fe2706c5b13e060d37e6b 2024-11-10T03:18:58,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/A/c146dc10e17441a58ee3d9a5655cdf4f, store: [table=TestAcidGuarantees family=A region=07b306bf509fe2706c5b13e060d37e6b] 2024-11-10T03:18:58,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/A/c146dc10e17441a58ee3d9a5655cdf4f is 175, key is test_row_0/A:col10/1731208737317/Put/seqid=0 2024-11-10T03:18:58,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742425_1601 (size=31105) 2024-11-10T03:18:58,294 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=160, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/A/c146dc10e17441a58ee3d9a5655cdf4f 2024-11-10T03:18:58,297 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:58,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55616 deadline: 1731208798294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:58,298 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:58,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55562 deadline: 1731208798294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:58,298 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:58,298 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:58,298 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:58,298 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:58,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55578 deadline: 1731208798294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:58,298 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:58,298 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:58,298 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:58,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:58,298 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:58,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55626 deadline: 1731208798294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:58,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:58,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:58,299 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:58,299 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:58,299 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:58,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/B/4defd168fa114b55800e15bc259371f0 is 50, key is test_row_0/B:col10/1731208737317/Put/seqid=0 2024-11-10T03:18:58,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742426_1602 (size=12151) 2024-11-10T03:18:58,330 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/B/4defd168fa114b55800e15bc259371f0 2024-11-10T03:18:58,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/C/8a344ce12dbd4c11a98afc3ce5351a8c is 50, key is test_row_0/C:col10/1731208737317/Put/seqid=0 2024-11-10T03:18:58,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742427_1603 (size=12151) 2024-11-10T03:18:58,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-10T03:18:58,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:58,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55616 deadline: 1731208798603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:58,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:58,606 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:58,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55562 deadline: 1731208798603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:58,606 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:58,606 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:58,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:58,606 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:58,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55578 deadline: 1731208798603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:58,606 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:58,606 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:58,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:58,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:58,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:58,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:58,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55626 deadline: 1731208798603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:58,607 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:58,607 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:58,607 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:58,753 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/C/8a344ce12dbd4c11a98afc3ce5351a8c 2024-11-10T03:18:58,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/A/c146dc10e17441a58ee3d9a5655cdf4f as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/c146dc10e17441a58ee3d9a5655cdf4f 2024-11-10T03:18:58,784 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/c146dc10e17441a58ee3d9a5655cdf4f, entries=150, sequenceid=160, filesize=30.4 K 2024-11-10T03:18:58,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/B/4defd168fa114b55800e15bc259371f0 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/4defd168fa114b55800e15bc259371f0 2024-11-10T03:18:58,791 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/4defd168fa114b55800e15bc259371f0, entries=150, sequenceid=160, filesize=11.9 K 2024-11-10T03:18:58,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/C/8a344ce12dbd4c11a98afc3ce5351a8c as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/8a344ce12dbd4c11a98afc3ce5351a8c 2024-11-10T03:18:58,801 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/8a344ce12dbd4c11a98afc3ce5351a8c, entries=150, sequenceid=160, filesize=11.9 K 2024-11-10T03:18:58,802 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 07b306bf509fe2706c5b13e060d37e6b in 936ms, sequenceid=160, compaction requested=true 2024-11-10T03:18:58,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for 07b306bf509fe2706c5b13e060d37e6b: 2024-11-10T03:18:58,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:18:58,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-11-10T03:18:58,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-11-10T03:18:58,807 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-10T03:18:58,807 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4070 sec 2024-11-10T03:18:58,809 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 1.4140 sec 2024-11-10T03:18:59,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on 07b306bf509fe2706c5b13e060d37e6b 2024-11-10T03:18:59,115 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 07b306bf509fe2706c5b13e060d37e6b 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-10T03:18:59,115 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07b306bf509fe2706c5b13e060d37e6b, store=A 2024-11-10T03:18:59,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:18:59,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07b306bf509fe2706c5b13e060d37e6b, store=B 2024-11-10T03:18:59,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:18:59,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07b306bf509fe2706c5b13e060d37e6b, store=C 2024-11-10T03:18:59,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:18:59,125 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111084e7f10886084f61b577aa7dfdf902d2_07b306bf509fe2706c5b13e060d37e6b is 50, key is test_row_0/A:col10/1731208737967/Put/seqid=0 2024-11-10T03:18:59,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:59,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55562 deadline: 1731208799129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:59,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:59,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:59,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:59,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742428_1604 (size=14794) 2024-11-10T03:18:59,135 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:18:59,138 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111084e7f10886084f61b577aa7dfdf902d2_07b306bf509fe2706c5b13e060d37e6b to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111084e7f10886084f61b577aa7dfdf902d2_07b306bf509fe2706c5b13e060d37e6b 2024-11-10T03:18:59,138 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:59,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55626 deadline: 1731208799132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:59,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:59,139 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:59,139 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:59,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55616 deadline: 1731208799132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:59,139 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:59,139 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:59,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55578 deadline: 1731208799133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:59,139 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:59,139 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:59,139 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:59,139 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/A/264b78c89cbf45d499500b81ce3304a2, store: [table=TestAcidGuarantees family=A region=07b306bf509fe2706c5b13e060d37e6b] 2024-11-10T03:18:59,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:59,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:59,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:59,140 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/A/264b78c89cbf45d499500b81ce3304a2 is 175, key is test_row_0/A:col10/1731208737967/Put/seqid=0 2024-11-10T03:18:59,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742429_1605 (size=39749) 2024-11-10T03:18:59,246 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:59,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55562 deadline: 1731208799242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:59,246 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:59,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:59,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55626 deadline: 1731208799243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:59,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:59,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:59,246 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:59,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55616 deadline: 1731208799243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:59,246 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:59,246 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:59,246 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:59,247 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:59,247 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:59,247 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:59,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:59,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55578 deadline: 1731208799243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:59,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:59,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:59,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:59,444 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:59,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55548 deadline: 1731208799437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:59,445 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 394d2222f724,38139,1731208572054 for 'test_row_2' in 07b306bf509fe2706c5b13e060d37e6b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T03:18:59,446 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:59,446 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:59,446 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:59,458 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:59,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55562 deadline: 1731208799453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:59,458 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:59,458 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:59,458 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:59,459 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:59,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55626 deadline: 1731208799453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:59,459 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:59,459 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:59,459 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:59,460 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:59,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55578 deadline: 1731208799453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:59,460 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:59,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55616 deadline: 1731208799453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:59,461 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:59,461 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:59,461 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:59,461 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:59,461 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:59,461 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:59,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-10T03:18:59,543 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T03:18:59,543 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T03:18:59,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-11-10T03:18:59,546 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T03:18:59,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-10T03:18:59,547 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T03:18:59,547 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T03:18:59,550 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=177, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/A/264b78c89cbf45d499500b81ce3304a2 2024-11-10T03:18:59,568 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/B/c4970472e33446819b39dbf2662298a5 is 50, key is test_row_0/B:col10/1731208737967/Put/seqid=0 2024-11-10T03:18:59,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742430_1606 (size=12151) 2024-11-10T03:18:59,602 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/B/c4970472e33446819b39dbf2662298a5 2024-11-10T03:18:59,610 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/C/46e735407cfe40f1b3cf447f524dade9 is 50, key is test_row_0/C:col10/1731208737967/Put/seqid=0 2024-11-10T03:18:59,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742431_1607 (size=12151) 2024-11-10T03:18:59,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-10T03:18:59,654 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/C/46e735407cfe40f1b3cf447f524dade9 2024-11-10T03:18:59,658 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/A/264b78c89cbf45d499500b81ce3304a2 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/264b78c89cbf45d499500b81ce3304a2 2024-11-10T03:18:59,664 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/264b78c89cbf45d499500b81ce3304a2, entries=200, sequenceid=177, filesize=38.8 K 2024-11-10T03:18:59,666 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/B/c4970472e33446819b39dbf2662298a5 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/c4970472e33446819b39dbf2662298a5 2024-11-10T03:18:59,674 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/c4970472e33446819b39dbf2662298a5, entries=150, sequenceid=177, filesize=11.9 K 2024-11-10T03:18:59,704 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-10T03:18:59,704 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/C/46e735407cfe40f1b3cf447f524dade9 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/46e735407cfe40f1b3cf447f524dade9 2024-11-10T03:18:59,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:18:59,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. as already flushing 2024-11-10T03:18:59,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:18:59,705 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:59,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:59,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:59,716 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/46e735407cfe40f1b3cf447f524dade9, entries=150, sequenceid=177, filesize=11.9 K 2024-11-10T03:18:59,717 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 07b306bf509fe2706c5b13e060d37e6b in 602ms, sequenceid=177, compaction requested=true 2024-11-10T03:18:59,717 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 07b306bf509fe2706c5b13e060d37e6b: 2024-11-10T03:18:59,717 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T03:18:59,718 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 07b306bf509fe2706c5b13e060d37e6b:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T03:18:59,718 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:18:59,718 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T03:18:59,718 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 150506 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T03:18:59,718 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 07b306bf509fe2706c5b13e060d37e6b:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T03:18:59,718 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:18:59,718 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 07b306bf509fe2706c5b13e060d37e6b:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T03:18:59,718 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1541): 07b306bf509fe2706c5b13e060d37e6b/A is initiating minor compaction (all files) 2024-11-10T03:18:59,718 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T03:18:59,719 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 07b306bf509fe2706c5b13e060d37e6b/A in TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:18:59,719 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/685288664520438eb094b998765b5264, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/e9322848bab7494cbc8d7e60323f53ba, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/c146dc10e17441a58ee3d9a5655cdf4f, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/264b78c89cbf45d499500b81ce3304a2] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp, totalSize=147.0 K 2024-11-10T03:18:59,719 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:18:59,719 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. files: [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/685288664520438eb094b998765b5264, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/e9322848bab7494cbc8d7e60323f53ba, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/c146dc10e17441a58ee3d9a5655cdf4f, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/264b78c89cbf45d499500b81ce3304a2] 2024-11-10T03:18:59,719 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 685288664520438eb094b998765b5264, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=124, earliestPutTs=1731208735247 2024-11-10T03:18:59,720 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48762 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T03:18:59,720 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting e9322848bab7494cbc8d7e60323f53ba, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1731208736146 2024-11-10T03:18:59,720 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1541): 07b306bf509fe2706c5b13e060d37e6b/B is initiating minor compaction (all files) 2024-11-10T03:18:59,720 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 07b306bf509fe2706c5b13e060d37e6b/B in TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:18:59,720 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting c146dc10e17441a58ee3d9a5655cdf4f, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1731208737312 2024-11-10T03:18:59,720 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/fafa0676e1944bb5a4886af093d6f596, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/49a3aa1e4c82453f856b12109cc877dd, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/4defd168fa114b55800e15bc259371f0, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/c4970472e33446819b39dbf2662298a5] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp, totalSize=47.6 K 2024-11-10T03:18:59,720 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting fafa0676e1944bb5a4886af093d6f596, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=124, earliestPutTs=1731208735247 2024-11-10T03:18:59,720 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 264b78c89cbf45d499500b81ce3304a2, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731208737957 2024-11-10T03:18:59,720 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 49a3aa1e4c82453f856b12109cc877dd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1731208736156 2024-11-10T03:18:59,721 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 4defd168fa114b55800e15bc259371f0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1731208737312 2024-11-10T03:18:59,721 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting c4970472e33446819b39dbf2662298a5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731208737957 2024-11-10T03:18:59,739 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=07b306bf509fe2706c5b13e060d37e6b] 2024-11-10T03:18:59,759 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 07b306bf509fe2706c5b13e060d37e6b#B#compaction#511 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:18:59,760 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/B/20eaa08314c845b0a434b7427807a0ac is 50, key is test_row_0/B:col10/1731208737967/Put/seqid=0 2024-11-10T03:18:59,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on 07b306bf509fe2706c5b13e060d37e6b 2024-11-10T03:18:59,768 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 07b306bf509fe2706c5b13e060d37e6b 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-10T03:18:59,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07b306bf509fe2706c5b13e060d37e6b, store=A 2024-11-10T03:18:59,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:18:59,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07b306bf509fe2706c5b13e060d37e6b, store=B 2024-11-10T03:18:59,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:18:59,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07b306bf509fe2706c5b13e060d37e6b, store=C 2024-11-10T03:18:59,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:18:59,773 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241110d21bd0129c6f47ac8298694d8ed2e636_07b306bf509fe2706c5b13e060d37e6b store=[table=TestAcidGuarantees family=A region=07b306bf509fe2706c5b13e060d37e6b] 2024-11-10T03:18:59,776 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241110d21bd0129c6f47ac8298694d8ed2e636_07b306bf509fe2706c5b13e060d37e6b, store=[table=TestAcidGuarantees family=A region=07b306bf509fe2706c5b13e060d37e6b] 2024-11-10T03:18:59,776 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110d21bd0129c6f47ac8298694d8ed2e636_07b306bf509fe2706c5b13e060d37e6b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=07b306bf509fe2706c5b13e060d37e6b] 2024-11-10T03:18:59,801 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110ff3ffca112ef4679948dbd7876dddd0d_07b306bf509fe2706c5b13e060d37e6b is 50, key is test_row_0/A:col10/1731208739125/Put/seqid=0 2024-11-10T03:18:59,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:59,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55562 deadline: 1731208799815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:59,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:59,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55626 deadline: 1731208799817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:59,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:59,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:59,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:59,825 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:59,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55578 deadline: 1731208799818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:59,825 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:59,825 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:59,825 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:59,825 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:59,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55616 deadline: 1731208799818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:59,826 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:59,826 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:59,826 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:59,826 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:59,826 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:59,826 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:59,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742432_1608 (size=12595) 2024-11-10T03:18:59,842 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/B/20eaa08314c845b0a434b7427807a0ac as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/20eaa08314c845b0a434b7427807a0ac 2024-11-10T03:18:59,848 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 07b306bf509fe2706c5b13e060d37e6b/B of 07b306bf509fe2706c5b13e060d37e6b into 20eaa08314c845b0a434b7427807a0ac(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:18:59,848 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 07b306bf509fe2706c5b13e060d37e6b: 2024-11-10T03:18:59,848 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., storeName=07b306bf509fe2706c5b13e060d37e6b/B, priority=12, startTime=1731208739718; duration=0sec 2024-11-10T03:18:59,848 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T03:18:59,848 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 07b306bf509fe2706c5b13e060d37e6b:B 2024-11-10T03:18:59,849 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T03:18:59,851 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48762 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T03:18:59,851 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1541): 07b306bf509fe2706c5b13e060d37e6b/C is initiating minor compaction (all files) 2024-11-10T03:18:59,851 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 07b306bf509fe2706c5b13e060d37e6b/C in TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:18:59,851 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/d4e9e04dafeb4e63ace5b0879c745b2c, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/5a2116a0c4364a12b011d795caf31789, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/8a344ce12dbd4c11a98afc3ce5351a8c, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/46e735407cfe40f1b3cf447f524dade9] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp, totalSize=47.6 K 2024-11-10T03:18:59,852 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting d4e9e04dafeb4e63ace5b0879c745b2c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=124, earliestPutTs=1731208735247 2024-11-10T03:18:59,852 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a2116a0c4364a12b011d795caf31789, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1731208736156 2024-11-10T03:18:59,852 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a344ce12dbd4c11a98afc3ce5351a8c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1731208737312 2024-11-10T03:18:59,852 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 46e735407cfe40f1b3cf447f524dade9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731208737957 2024-11-10T03:18:59,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742433_1609 (size=4469) 2024-11-10T03:18:59,855 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 07b306bf509fe2706c5b13e060d37e6b#A#compaction#510 average throughput is 0.21 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:18:59,856 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/A/f9ba9aed7ae749509794b7b2dc2c1554 is 175, key is test_row_0/A:col10/1731208737967/Put/seqid=0 2024-11-10T03:18:59,860 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-10T03:18:59,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:18:59,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. as already flushing 2024-11-10T03:18:59,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:18:59,861 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:59,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:59,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:18:59,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-10T03:18:59,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742434_1610 (size=14794) 2024-11-10T03:18:59,879 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:18:59,884 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110ff3ffca112ef4679948dbd7876dddd0d_07b306bf509fe2706c5b13e060d37e6b to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110ff3ffca112ef4679948dbd7876dddd0d_07b306bf509fe2706c5b13e060d37e6b 2024-11-10T03:18:59,887 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/A/d9a3a63a766d4a7486b5df08d3574d93, store: [table=TestAcidGuarantees family=A region=07b306bf509fe2706c5b13e060d37e6b] 2024-11-10T03:18:59,888 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/A/d9a3a63a766d4a7486b5df08d3574d93 is 175, key is test_row_0/A:col10/1731208739125/Put/seqid=0 2024-11-10T03:18:59,919 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 07b306bf509fe2706c5b13e060d37e6b#C#compaction#513 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:18:59,919 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/C/c380fc8373304f088f44503c03e34689 is 50, key is test_row_0/C:col10/1731208737967/Put/seqid=0 2024-11-10T03:18:59,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742436_1612 (size=39749) 2024-11-10T03:18:59,925 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=194, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/A/d9a3a63a766d4a7486b5df08d3574d93 2024-11-10T03:18:59,935 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:59,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55562 deadline: 1731208799933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:59,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742435_1611 (size=31549) 2024-11-10T03:18:59,935 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:59,935 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:59,935 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:59,935 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:59,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55626 deadline: 1731208799933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:59,935 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:59,935 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:59,935 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:59,935 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:59,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55616 deadline: 1731208799933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:59,936 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:59,936 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:59,936 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:59,936 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:18:59,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55578 deadline: 1731208799933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:18:59,936 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:59,936 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:18:59,936 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:18:59,938 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/A/f9ba9aed7ae749509794b7b2dc2c1554 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/f9ba9aed7ae749509794b7b2dc2c1554 2024-11-10T03:18:59,942 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 07b306bf509fe2706c5b13e060d37e6b/A of 07b306bf509fe2706c5b13e060d37e6b into f9ba9aed7ae749509794b7b2dc2c1554(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:18:59,942 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 07b306bf509fe2706c5b13e060d37e6b: 2024-11-10T03:18:59,942 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., storeName=07b306bf509fe2706c5b13e060d37e6b/A, priority=12, startTime=1731208739717; duration=0sec 2024-11-10T03:18:59,942 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:18:59,942 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 07b306bf509fe2706c5b13e060d37e6b:A 2024-11-10T03:18:59,946 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/B/abc3e04fcd244982880cda8c79b4cb7d is 50, key is test_row_0/B:col10/1731208739125/Put/seqid=0 2024-11-10T03:18:59,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742437_1613 (size=12595) 2024-11-10T03:18:59,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742438_1614 (size=12151) 2024-11-10T03:18:59,949 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/B/abc3e04fcd244982880cda8c79b4cb7d 2024-11-10T03:18:59,956 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/C/6c9a604eebb24e97ab2df45b743e75fc is 50, key is test_row_0/C:col10/1731208739125/Put/seqid=0 2024-11-10T03:18:59,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742439_1615 (size=12151) 2024-11-10T03:18:59,959 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/C/6c9a604eebb24e97ab2df45b743e75fc 2024-11-10T03:18:59,962 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/A/d9a3a63a766d4a7486b5df08d3574d93 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/d9a3a63a766d4a7486b5df08d3574d93 2024-11-10T03:18:59,965 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/d9a3a63a766d4a7486b5df08d3574d93, entries=200, sequenceid=194, filesize=38.8 K 2024-11-10T03:18:59,965 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/B/abc3e04fcd244982880cda8c79b4cb7d as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/abc3e04fcd244982880cda8c79b4cb7d 2024-11-10T03:18:59,968 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/abc3e04fcd244982880cda8c79b4cb7d, entries=150, sequenceid=194, filesize=11.9 K 2024-11-10T03:18:59,972 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/C/6c9a604eebb24e97ab2df45b743e75fc as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/6c9a604eebb24e97ab2df45b743e75fc 2024-11-10T03:18:59,979 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/6c9a604eebb24e97ab2df45b743e75fc, entries=150, sequenceid=194, filesize=11.9 K 2024-11-10T03:18:59,981 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 07b306bf509fe2706c5b13e060d37e6b in 213ms, sequenceid=194, compaction requested=false 2024-11-10T03:18:59,981 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 07b306bf509fe2706c5b13e060d37e6b: 2024-11-10T03:19:00,014 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-10T03:19:00,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:19:00,014 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing 07b306bf509fe2706c5b13e060d37e6b 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-10T03:19:00,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07b306bf509fe2706c5b13e060d37e6b, store=A 2024-11-10T03:19:00,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:19:00,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07b306bf509fe2706c5b13e060d37e6b, store=B 2024-11-10T03:19:00,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:19:00,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07b306bf509fe2706c5b13e060d37e6b, store=C 2024-11-10T03:19:00,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:19:00,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411103116b00f72594f3aafcfa5fd4cf5617e_07b306bf509fe2706c5b13e060d37e6b is 50, key is test_row_0/A:col10/1731208739816/Put/seqid=0 2024-11-10T03:19:00,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742440_1616 (size=12304) 2024-11-10T03:19:00,147 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. as already flushing 2024-11-10T03:19:00,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on 07b306bf509fe2706c5b13e060d37e6b 2024-11-10T03:19:00,170 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:19:00,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55562 deadline: 1731208800162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:19:00,170 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:00,170 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:00,170 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:19:00,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-10T03:19:00,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:19:00,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55616 deadline: 1731208800169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:19:00,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:19:00,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55578 deadline: 1731208800170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:19:00,175 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:19:00,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55626 deadline: 1731208800171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:19:00,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:00,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:00,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:19:00,175 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:00,175 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:00,175 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:19:00,175 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:00,175 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:00,176 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:19:00,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:19:00,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55562 deadline: 1731208800272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:19:00,274 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:00,275 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:00,275 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:19:00,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:19:00,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55578 deadline: 1731208800283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:19:00,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:19:00,286 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:00,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55626 deadline: 1731208800283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:19:00,286 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:00,286 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:19:00,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:19:00,286 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:00,286 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:00,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55616 deadline: 1731208800283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:19:00,286 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:19:00,286 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:00,286 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:00,286 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:19:00,352 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/C/c380fc8373304f088f44503c03e34689 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/c380fc8373304f088f44503c03e34689 2024-11-10T03:19:00,372 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 07b306bf509fe2706c5b13e060d37e6b/C of 07b306bf509fe2706c5b13e060d37e6b into c380fc8373304f088f44503c03e34689(size=12.3 K), total size for store is 24.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:19:00,372 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 07b306bf509fe2706c5b13e060d37e6b: 2024-11-10T03:19:00,372 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., storeName=07b306bf509fe2706c5b13e060d37e6b/C, priority=12, startTime=1731208739718; duration=0sec 2024-11-10T03:19:00,372 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:19:00,372 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 07b306bf509fe2706c5b13e060d37e6b:C 2024-11-10T03:19:00,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:19:00,439 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411103116b00f72594f3aafcfa5fd4cf5617e_07b306bf509fe2706c5b13e060d37e6b to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411103116b00f72594f3aafcfa5fd4cf5617e_07b306bf509fe2706c5b13e060d37e6b 2024-11-10T03:19:00,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/A/860047d56dde470c8b0af3cc502ac655, store: [table=TestAcidGuarantees family=A region=07b306bf509fe2706c5b13e060d37e6b] 2024-11-10T03:19:00,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/A/860047d56dde470c8b0af3cc502ac655 is 175, key is test_row_0/A:col10/1731208739816/Put/seqid=0 2024-11-10T03:19:00,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742441_1617 (size=31105) 2024-11-10T03:19:00,484 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:19:00,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55562 deadline: 1731208800483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:19:00,485 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:00,485 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:00,485 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:19:00,495 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:19:00,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55578 deadline: 1731208800492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:19:00,495 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:19:00,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55626 deadline: 1731208800493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:19:00,495 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:00,495 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:00,496 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:19:00,496 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:00,496 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:19:00,496 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:00,496 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:19:00,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55616 deadline: 1731208800493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:19:00,496 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:00,496 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:00,496 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:19:00,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-10T03:19:00,796 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:19:00,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55562 deadline: 1731208800793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:19:00,796 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:00,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:00,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:19:00,807 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:19:00,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55578 deadline: 1731208800803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:19:00,807 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:19:00,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:00,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:00,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55626 deadline: 1731208800803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:19:00,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:19:00,808 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:00,808 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:00,808 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:19:00,808 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:19:00,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55616 deadline: 1731208800803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:19:00,808 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:00,808 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:00,808 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:19:00,845 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=211, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/A/860047d56dde470c8b0af3cc502ac655 2024-11-10T03:19:00,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/B/c377e74ec4ce4f53b86de58c3056d232 is 50, key is test_row_0/B:col10/1731208739816/Put/seqid=0 2024-11-10T03:19:00,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742442_1618 (size=12151) 2024-11-10T03:19:01,270 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/B/c377e74ec4ce4f53b86de58c3056d232 2024-11-10T03:19:01,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/C/7aefeb063ed94d658f88bf62ddbcf0e9 is 50, key is test_row_0/C:col10/1731208739816/Put/seqid=0 2024-11-10T03:19:01,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742443_1619 (size=12151) 2024-11-10T03:19:01,306 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/C/7aefeb063ed94d658f88bf62ddbcf0e9 2024-11-10T03:19:01,307 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:19:01,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55562 deadline: 1731208801303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:19:01,308 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:01,308 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:01,308 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:19:01,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/A/860047d56dde470c8b0af3cc502ac655 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/860047d56dde470c8b0af3cc502ac655 2024-11-10T03:19:01,312 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/860047d56dde470c8b0af3cc502ac655, entries=150, sequenceid=211, filesize=30.4 K 2024-11-10T03:19:01,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/B/c377e74ec4ce4f53b86de58c3056d232 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/c377e74ec4ce4f53b86de58c3056d232 2024-11-10T03:19:01,315 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:19:01,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55616 deadline: 1731208801313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:19:01,316 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:01,316 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:01,316 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:19:01,316 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:19:01,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55578 deadline: 1731208801313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:19:01,316 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:01,316 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:19:01,316 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:01,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55626 deadline: 1731208801313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:19:01,316 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:19:01,317 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:01,317 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:01,317 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:19:01,317 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/c377e74ec4ce4f53b86de58c3056d232, entries=150, sequenceid=211, filesize=11.9 K 2024-11-10T03:19:01,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/C/7aefeb063ed94d658f88bf62ddbcf0e9 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/7aefeb063ed94d658f88bf62ddbcf0e9 2024-11-10T03:19:01,328 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/7aefeb063ed94d658f88bf62ddbcf0e9, entries=150, sequenceid=211, filesize=11.9 K 2024-11-10T03:19:01,328 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for 07b306bf509fe2706c5b13e060d37e6b in 1314ms, sequenceid=211, compaction requested=true 2024-11-10T03:19:01,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for 07b306bf509fe2706c5b13e060d37e6b: 2024-11-10T03:19:01,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:19:01,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-11-10T03:19:01,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-11-10T03:19:01,332 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-10T03:19:01,332 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7830 sec 2024-11-10T03:19:01,333 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 1.7890 sec 2024-11-10T03:19:01,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-10T03:19:01,693 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T03:19:01,693 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T03:19:01,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees 2024-11-10T03:19:01,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-10T03:19:01,695 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T03:19:01,695 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T03:19:01,696 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T03:19:01,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-10T03:19:01,848 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-10T03:19:01,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:19:01,848 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2902): Flushing 07b306bf509fe2706c5b13e060d37e6b 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-10T03:19:01,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07b306bf509fe2706c5b13e060d37e6b, store=A 2024-11-10T03:19:01,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:19:01,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07b306bf509fe2706c5b13e060d37e6b, store=B 2024-11-10T03:19:01,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:19:01,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07b306bf509fe2706c5b13e060d37e6b, store=C 2024-11-10T03:19:01,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:19:01,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111072d171f7377c4adf9803995cd510ada6_07b306bf509fe2706c5b13e060d37e6b is 50, key is test_row_0/A:col10/1731208740169/Put/seqid=0 2024-11-10T03:19:01,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742444_1620 (size=12304) 2024-11-10T03:19:01,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:19:01,932 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111072d171f7377c4adf9803995cd510ada6_07b306bf509fe2706c5b13e060d37e6b to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111072d171f7377c4adf9803995cd510ada6_07b306bf509fe2706c5b13e060d37e6b 2024-11-10T03:19:01,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/A/3afb1fe1836c476f8741dea4412bbae5, store: [table=TestAcidGuarantees family=A region=07b306bf509fe2706c5b13e060d37e6b] 2024-11-10T03:19:01,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/A/3afb1fe1836c476f8741dea4412bbae5 is 175, key is test_row_0/A:col10/1731208740169/Put/seqid=0 2024-11-10T03:19:01,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742445_1621 (size=31105) 2024-11-10T03:19:02,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-10T03:19:02,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-10T03:19:02,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on 07b306bf509fe2706c5b13e060d37e6b 2024-11-10T03:19:02,326 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. as already flushing 2024-11-10T03:19:02,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:19:02,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55562 deadline: 1731208802341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:19:02,344 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:02,344 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:02,344 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:19:02,347 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:19:02,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55626 deadline: 1731208802342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:19:02,347 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:02,347 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:02,347 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:19:02,348 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=230, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/A/3afb1fe1836c476f8741dea4412bbae5 2024-11-10T03:19:02,350 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:19:02,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55616 deadline: 1731208802343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:19:02,350 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:02,350 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:19:02,350 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:02,350 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:19:02,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55578 deadline: 1731208802344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:19:02,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:02,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:02,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:19:02,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/B/9bd9781a72784d23bb675e83a9b4f1ba is 50, key is test_row_0/B:col10/1731208740169/Put/seqid=0 2024-11-10T03:19:02,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742446_1622 (size=12151) 2024-11-10T03:19:02,356 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/B/9bd9781a72784d23bb675e83a9b4f1ba 2024-11-10T03:19:02,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/C/9fb2fa6ce3554e2d8a1cab39114a760c is 50, key is test_row_0/C:col10/1731208740169/Put/seqid=0 2024-11-10T03:19:02,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742447_1623 (size=12151) 2024-11-10T03:19:02,457 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:19:02,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55562 deadline: 1731208802453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:19:02,458 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:02,458 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:19:02,458 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:02,458 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:19:02,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55626 deadline: 1731208802453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:19:02,458 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:02,458 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:02,458 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:19:02,458 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:19:02,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55616 deadline: 1731208802453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:19:02,458 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:02,458 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:02,458 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:19:02,458 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:19:02,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55578 deadline: 1731208802453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:19:02,459 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:02,459 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:02,459 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:19:02,665 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:19:02,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55562 deadline: 1731208802663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:19:02,665 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:02,666 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:02,666 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:19:02,668 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:19:02,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55578 deadline: 1731208802663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:19:02,669 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:02,669 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:19:02,669 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:19:02,669 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:02,669 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:19:02,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55616 deadline: 1731208802663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:19:02,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55626 deadline: 1731208802663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:19:02,669 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:02,669 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:02,669 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:19:02,669 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:02,669 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:02,669 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:19:02,772 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/C/9fb2fa6ce3554e2d8a1cab39114a760c 2024-11-10T03:19:02,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/A/3afb1fe1836c476f8741dea4412bbae5 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/3afb1fe1836c476f8741dea4412bbae5 2024-11-10T03:19:02,783 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/3afb1fe1836c476f8741dea4412bbae5, entries=150, sequenceid=230, filesize=30.4 K 2024-11-10T03:19:02,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/B/9bd9781a72784d23bb675e83a9b4f1ba as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/9bd9781a72784d23bb675e83a9b4f1ba 2024-11-10T03:19:02,787 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/9bd9781a72784d23bb675e83a9b4f1ba, entries=150, sequenceid=230, filesize=11.9 K 2024-11-10T03:19:02,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/C/9fb2fa6ce3554e2d8a1cab39114a760c as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/9fb2fa6ce3554e2d8a1cab39114a760c 2024-11-10T03:19:02,793 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/9fb2fa6ce3554e2d8a1cab39114a760c, entries=150, sequenceid=230, filesize=11.9 K 2024-11-10T03:19:02,794 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for 07b306bf509fe2706c5b13e060d37e6b in 946ms, sequenceid=230, compaction requested=true 2024-11-10T03:19:02,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2603): Flush status journal for 07b306bf509fe2706c5b13e060d37e6b: 2024-11-10T03:19:02,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:19:02,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=165 2024-11-10T03:19:02,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4169): Remote procedure done, pid=165 2024-11-10T03:19:02,798 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-10T03:19:02,800 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0990 sec 2024-11-10T03:19:02,802 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees in 1.1060 sec 2024-11-10T03:19:02,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-10T03:19:02,833 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T03:19:02,833 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T03:19:02,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees 2024-11-10T03:19:02,835 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T03:19:02,836 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T03:19:02,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-10T03:19:02,836 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T03:19:02,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-10T03:19:02,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on 07b306bf509fe2706c5b13e060d37e6b 2024-11-10T03:19:02,976 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 07b306bf509fe2706c5b13e060d37e6b 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-10T03:19:02,977 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07b306bf509fe2706c5b13e060d37e6b, store=A 2024-11-10T03:19:02,977 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:19:02,977 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07b306bf509fe2706c5b13e060d37e6b, store=B 2024-11-10T03:19:02,977 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:19:02,977 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07b306bf509fe2706c5b13e060d37e6b, store=C 2024-11-10T03:19:02,977 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:19:02,988 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-10T03:19:02,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:19:02,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. as already flushing 2024-11-10T03:19:02,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:19:02,989 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:19:02,989 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411103d1ea80acb2f478882e709e4f34524aa_07b306bf509fe2706c5b13e060d37e6b is 50, key is test_row_0/A:col10/1731208742341/Put/seqid=0 2024-11-10T03:19:02,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:19:02,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:19:03,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742448_1624 (size=17284) 2024-11-10T03:19:03,006 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:19:03,007 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:19:03,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55578 deadline: 1731208803001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:19:03,007 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:03,007 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:03,007 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:19:03,007 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:19:03,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55562 deadline: 1731208803002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:19:03,008 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:03,008 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:19:03,008 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:03,008 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:19:03,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55626 deadline: 1731208803002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:19:03,008 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:03,009 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:03,009 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:19:03,010 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411103d1ea80acb2f478882e709e4f34524aa_07b306bf509fe2706c5b13e060d37e6b to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411103d1ea80acb2f478882e709e4f34524aa_07b306bf509fe2706c5b13e060d37e6b 2024-11-10T03:19:03,010 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/A/259bfaf7b8ce43a6a90507168277f930, store: [table=TestAcidGuarantees family=A region=07b306bf509fe2706c5b13e060d37e6b] 2024-11-10T03:19:03,011 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/A/259bfaf7b8ce43a6a90507168277f930 is 175, key is test_row_0/A:col10/1731208742341/Put/seqid=0 2024-11-10T03:19:03,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742449_1625 (size=48389) 2024-11-10T03:19:03,016 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=247, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/A/259bfaf7b8ce43a6a90507168277f930 2024-11-10T03:19:03,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:19:03,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55616 deadline: 1731208803012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:19:03,017 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:03,017 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:03,018 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:19:03,066 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/B/cd5f885188244e999b4e6c5a7ee5a1cb is 50, key is test_row_0/B:col10/1731208742341/Put/seqid=0 2024-11-10T03:19:03,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742450_1626 (size=12151) 2024-11-10T03:19:03,117 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:19:03,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55578 deadline: 1731208803112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:19:03,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:19:03,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55562 deadline: 1731208803113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:19:03,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:19:03,118 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:03,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55626 deadline: 1731208803113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:19:03,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:03,118 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:03,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:03,118 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:19:03,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:19:03,118 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:03,118 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:03,118 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:19:03,128 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:19:03,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55616 deadline: 1731208803123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:19:03,129 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:03,129 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:03,129 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:19:03,143 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-10T03:19:03,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:19:03,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. as already flushing 2024-11-10T03:19:03,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:19:03,144 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:19:03,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:19:03,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:19:03,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-10T03:19:03,297 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-10T03:19:03,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:19:03,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. as already flushing 2024-11-10T03:19:03,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:19:03,297 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:19:03,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:19:03,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:19:03,327 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:19:03,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55578 deadline: 1731208803323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:19:03,327 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:19:03,328 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:03,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55562 deadline: 1731208803323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:19:03,328 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:03,328 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:19:03,328 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:19:03,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:03,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:03,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55626 deadline: 1731208803323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:19:03,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:19:03,328 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:03,328 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:03,328 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:19:03,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:19:03,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55616 deadline: 1731208803333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:19:03,336 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:03,336 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:03,336 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:19:03,449 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-10T03:19:03,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:19:03,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. as already flushing 2024-11-10T03:19:03,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:19:03,450 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:19:03,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:19:03,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:19:03,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-10T03:19:03,480 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/B/cd5f885188244e999b4e6c5a7ee5a1cb 2024-11-10T03:19:03,487 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/C/1e758f0108ae4175a0e6d4987291398b is 50, key is test_row_0/C:col10/1731208742341/Put/seqid=0 2024-11-10T03:19:03,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:19:03,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55548 deadline: 1731208803483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:19:03,488 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 394d2222f724,38139,1731208572054 for 'test_row_2' in 07b306bf509fe2706c5b13e060d37e6b of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8229 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T03:19:03,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:03,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:03,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:19:03,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742451_1627 (size=12151) 2024-11-10T03:19:03,602 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-10T03:19:03,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:19:03,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. as already flushing 2024-11-10T03:19:03,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:19:03,602 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:19:03,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:19:03,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:19:03,637 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:19:03,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55578 deadline: 1731208803633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:19:03,638 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:19:03,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55562 deadline: 1731208803633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:19:03,639 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:03,639 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:03,639 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:19:03,639 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:19:03,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55626 deadline: 1731208803634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:19:03,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:03,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:03,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:19:03,639 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:03,639 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:03,639 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:19:03,646 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:19:03,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55616 deadline: 1731208803643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:19:03,647 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:03,647 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:03,647 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:19:03,755 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-10T03:19:03,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:19:03,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. as already flushing 2024-11-10T03:19:03,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:19:03,755 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:19:03,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:19:03,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:19:03,892 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/C/1e758f0108ae4175a0e6d4987291398b 2024-11-10T03:19:03,895 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/A/259bfaf7b8ce43a6a90507168277f930 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/259bfaf7b8ce43a6a90507168277f930 2024-11-10T03:19:03,898 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/259bfaf7b8ce43a6a90507168277f930, entries=250, sequenceid=247, filesize=47.3 K 2024-11-10T03:19:03,899 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/B/cd5f885188244e999b4e6c5a7ee5a1cb as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/cd5f885188244e999b4e6c5a7ee5a1cb 2024-11-10T03:19:03,902 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/cd5f885188244e999b4e6c5a7ee5a1cb, entries=150, sequenceid=247, filesize=11.9 K 2024-11-10T03:19:03,903 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/C/1e758f0108ae4175a0e6d4987291398b as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/1e758f0108ae4175a0e6d4987291398b 2024-11-10T03:19:03,909 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/1e758f0108ae4175a0e6d4987291398b, entries=150, sequenceid=247, filesize=11.9 K 2024-11-10T03:19:03,911 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for 07b306bf509fe2706c5b13e060d37e6b in 935ms, sequenceid=247, compaction requested=true 2024-11-10T03:19:03,911 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 07b306bf509fe2706c5b13e060d37e6b: 2024-11-10T03:19:03,911 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 07b306bf509fe2706c5b13e060d37e6b:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T03:19:03,911 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:19:03,911 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-10T03:19:03,911 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 07b306bf509fe2706c5b13e060d37e6b:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T03:19:03,911 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:19:03,911 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-10T03:19:03,911 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 07b306bf509fe2706c5b13e060d37e6b:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T03:19:03,911 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T03:19:03,912 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-10T03:19:03,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:19:03,912 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2902): Flushing 07b306bf509fe2706c5b13e060d37e6b 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-10T03:19:03,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07b306bf509fe2706c5b13e060d37e6b, store=A 2024-11-10T03:19:03,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:19:03,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07b306bf509fe2706c5b13e060d37e6b, store=B 2024-11-10T03:19:03,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:19:03,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07b306bf509fe2706c5b13e060d37e6b, store=C 2024-11-10T03:19:03,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:19:03,913 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61199 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-10T03:19:03,913 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 181897 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-10T03:19:03,913 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1541): 07b306bf509fe2706c5b13e060d37e6b/B is initiating minor compaction (all files) 2024-11-10T03:19:03,913 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1541): 07b306bf509fe2706c5b13e060d37e6b/A is initiating minor compaction (all files) 2024-11-10T03:19:03,913 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 07b306bf509fe2706c5b13e060d37e6b/B in TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:19:03,913 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 07b306bf509fe2706c5b13e060d37e6b/A in TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:19:03,913 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/f9ba9aed7ae749509794b7b2dc2c1554, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/d9a3a63a766d4a7486b5df08d3574d93, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/860047d56dde470c8b0af3cc502ac655, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/3afb1fe1836c476f8741dea4412bbae5, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/259bfaf7b8ce43a6a90507168277f930] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp, totalSize=177.6 K 2024-11-10T03:19:03,913 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/20eaa08314c845b0a434b7427807a0ac, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/abc3e04fcd244982880cda8c79b4cb7d, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/c377e74ec4ce4f53b86de58c3056d232, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/9bd9781a72784d23bb675e83a9b4f1ba, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/cd5f885188244e999b4e6c5a7ee5a1cb] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp, totalSize=59.8 K 2024-11-10T03:19:03,913 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=11 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:19:03,913 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. files: [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/f9ba9aed7ae749509794b7b2dc2c1554, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/d9a3a63a766d4a7486b5df08d3574d93, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/860047d56dde470c8b0af3cc502ac655, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/3afb1fe1836c476f8741dea4412bbae5, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/259bfaf7b8ce43a6a90507168277f930] 2024-11-10T03:19:03,913 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 20eaa08314c845b0a434b7427807a0ac, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731208737957 2024-11-10T03:19:03,913 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting f9ba9aed7ae749509794b7b2dc2c1554, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731208737957 2024-11-10T03:19:03,914 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting abc3e04fcd244982880cda8c79b4cb7d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731208739125 2024-11-10T03:19:03,914 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting d9a3a63a766d4a7486b5df08d3574d93, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731208739125 2024-11-10T03:19:03,914 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 860047d56dde470c8b0af3cc502ac655, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731208739797 2024-11-10T03:19:03,914 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting c377e74ec4ce4f53b86de58c3056d232, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731208739797 2024-11-10T03:19:03,914 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3afb1fe1836c476f8741dea4412bbae5, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731208740160 2024-11-10T03:19:03,914 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 9bd9781a72784d23bb675e83a9b4f1ba, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731208740160 2024-11-10T03:19:03,914 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting cd5f885188244e999b4e6c5a7ee5a1cb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731208742341 2024-11-10T03:19:03,914 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 259bfaf7b8ce43a6a90507168277f930, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731208742336 2024-11-10T03:19:03,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110cb67ebdc18de45af9e001d883d16fac2_07b306bf509fe2706c5b13e060d37e6b is 50, key is test_row_0/A:col10/1731208743001/Put/seqid=0 2024-11-10T03:19:03,927 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 07b306bf509fe2706c5b13e060d37e6b#B#compaction#526 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:19:03,928 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/B/ac10ffb12c06408288e9228e0fdbc7b6 is 50, key is test_row_0/B:col10/1731208742341/Put/seqid=0 2024-11-10T03:19:03,928 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=07b306bf509fe2706c5b13e060d37e6b] 2024-11-10T03:19:03,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742452_1628 (size=12454) 2024-11-10T03:19:03,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:19:03,933 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110cb67ebdc18de45af9e001d883d16fac2_07b306bf509fe2706c5b13e060d37e6b to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110cb67ebdc18de45af9e001d883d16fac2_07b306bf509fe2706c5b13e060d37e6b 2024-11-10T03:19:03,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/A/0131d2f2ee024519a36b11cbbf0c9851, store: [table=TestAcidGuarantees family=A region=07b306bf509fe2706c5b13e060d37e6b] 2024-11-10T03:19:03,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/A/0131d2f2ee024519a36b11cbbf0c9851 is 175, key is test_row_0/A:col10/1731208743001/Put/seqid=0 2024-11-10T03:19:03,961 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241110c8db0d62c6ff4197b0f668b58b6c6378_07b306bf509fe2706c5b13e060d37e6b store=[table=TestAcidGuarantees family=A region=07b306bf509fe2706c5b13e060d37e6b] 2024-11-10T03:19:03,966 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241110c8db0d62c6ff4197b0f668b58b6c6378_07b306bf509fe2706c5b13e060d37e6b, store=[table=TestAcidGuarantees family=A region=07b306bf509fe2706c5b13e060d37e6b] 2024-11-10T03:19:03,966 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110c8db0d62c6ff4197b0f668b58b6c6378_07b306bf509fe2706c5b13e060d37e6b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=07b306bf509fe2706c5b13e060d37e6b] 2024-11-10T03:19:03,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-10T03:19:03,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742454_1630 (size=31255) 2024-11-10T03:19:03,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742453_1629 (size=12765) 2024-11-10T03:19:03,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742455_1631 (size=4469) 2024-11-10T03:19:03,989 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 07b306bf509fe2706c5b13e060d37e6b#A#compaction#527 average throughput is 0.40 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:19:03,989 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/A/5a5a824938bd4b5cb816103a25d4ae91 is 175, key is test_row_0/A:col10/1731208742341/Put/seqid=0 2024-11-10T03:19:04,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742456_1632 (size=31719) 2024-11-10T03:19:04,010 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/A/5a5a824938bd4b5cb816103a25d4ae91 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/5a5a824938bd4b5cb816103a25d4ae91 2024-11-10T03:19:04,014 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 07b306bf509fe2706c5b13e060d37e6b/A of 07b306bf509fe2706c5b13e060d37e6b into 5a5a824938bd4b5cb816103a25d4ae91(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:19:04,014 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 07b306bf509fe2706c5b13e060d37e6b: 2024-11-10T03:19:04,014 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., storeName=07b306bf509fe2706c5b13e060d37e6b/A, priority=11, startTime=1731208743911; duration=0sec 2024-11-10T03:19:04,015 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T03:19:04,015 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 07b306bf509fe2706c5b13e060d37e6b:A 2024-11-10T03:19:04,015 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-10T03:19:04,016 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61199 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-10T03:19:04,016 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1541): 07b306bf509fe2706c5b13e060d37e6b/C is initiating minor compaction (all files) 2024-11-10T03:19:04,016 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 07b306bf509fe2706c5b13e060d37e6b/C in TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:19:04,016 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/c380fc8373304f088f44503c03e34689, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/6c9a604eebb24e97ab2df45b743e75fc, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/7aefeb063ed94d658f88bf62ddbcf0e9, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/9fb2fa6ce3554e2d8a1cab39114a760c, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/1e758f0108ae4175a0e6d4987291398b] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp, totalSize=59.8 K 2024-11-10T03:19:04,016 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting c380fc8373304f088f44503c03e34689, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731208737957 2024-11-10T03:19:04,016 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c9a604eebb24e97ab2df45b743e75fc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731208739125 2024-11-10T03:19:04,017 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7aefeb063ed94d658f88bf62ddbcf0e9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731208739797 2024-11-10T03:19:04,018 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9fb2fa6ce3554e2d8a1cab39114a760c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731208740160 2024-11-10T03:19:04,018 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e758f0108ae4175a0e6d4987291398b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731208742341 2024-11-10T03:19:04,026 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 07b306bf509fe2706c5b13e060d37e6b#C#compaction#528 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:19:04,027 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/C/a1a5884f6c884c26bccc421cd31011fd is 50, key is test_row_0/C:col10/1731208742341/Put/seqid=0 2024-11-10T03:19:04,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742457_1633 (size=12765) 2024-11-10T03:19:04,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on 07b306bf509fe2706c5b13e060d37e6b 2024-11-10T03:19:04,145 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. as already flushing 2024-11-10T03:19:04,170 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:19:04,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55626 deadline: 1731208804164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:19:04,171 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:19:04,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55562 deadline: 1731208804165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:19:04,171 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:04,171 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:04,171 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:19:04,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:04,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:04,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:19:04,173 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:19:04,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55578 deadline: 1731208804169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:19:04,173 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:04,173 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:04,173 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:19:04,175 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:19:04,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55616 deadline: 1731208804171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:19:04,175 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:04,176 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:04,176 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:19:04,275 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:19:04,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55562 deadline: 1731208804273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:19:04,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:04,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:04,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:19:04,285 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:19:04,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55626 deadline: 1731208804283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:19:04,285 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:19:04,285 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:04,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55616 deadline: 1731208804283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:19:04,286 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:04,286 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:19:04,286 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:19:04,286 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:04,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55578 deadline: 1731208804283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:19:04,286 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:04,286 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:19:04,286 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:04,286 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:04,286 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:19:04,377 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=263, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/A/0131d2f2ee024519a36b11cbbf0c9851 2024-11-10T03:19:04,380 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/B/ac10ffb12c06408288e9228e0fdbc7b6 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/ac10ffb12c06408288e9228e0fdbc7b6 2024-11-10T03:19:04,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/B/ecf1d40a3d184a25b663072129b6f03c is 50, key is test_row_0/B:col10/1731208743001/Put/seqid=0 2024-11-10T03:19:04,389 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 07b306bf509fe2706c5b13e060d37e6b/B of 07b306bf509fe2706c5b13e060d37e6b into ac10ffb12c06408288e9228e0fdbc7b6(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:19:04,389 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 07b306bf509fe2706c5b13e060d37e6b: 2024-11-10T03:19:04,389 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., storeName=07b306bf509fe2706c5b13e060d37e6b/B, priority=11, startTime=1731208743911; duration=0sec 2024-11-10T03:19:04,389 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:19:04,389 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 07b306bf509fe2706c5b13e060d37e6b:B 2024-11-10T03:19:04,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742458_1634 (size=12301) 2024-11-10T03:19:04,400 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/B/ecf1d40a3d184a25b663072129b6f03c 2024-11-10T03:19:04,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/C/86b8f45779164e73abcb3ef7c1b466bd is 50, key is test_row_0/C:col10/1731208743001/Put/seqid=0 2024-11-10T03:19:04,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742459_1635 (size=12301) 2024-11-10T03:19:04,434 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/C/a1a5884f6c884c26bccc421cd31011fd as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/a1a5884f6c884c26bccc421cd31011fd 2024-11-10T03:19:04,438 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 07b306bf509fe2706c5b13e060d37e6b/C of 07b306bf509fe2706c5b13e060d37e6b into a1a5884f6c884c26bccc421cd31011fd(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:19:04,438 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 07b306bf509fe2706c5b13e060d37e6b: 2024-11-10T03:19:04,438 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., storeName=07b306bf509fe2706c5b13e060d37e6b/C, priority=11, startTime=1731208743911; duration=0sec 2024-11-10T03:19:04,438 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:19:04,438 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 07b306bf509fe2706c5b13e060d37e6b:C 2024-11-10T03:19:04,485 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:19:04,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55562 deadline: 1731208804482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:19:04,485 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:04,486 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:04,486 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:19:04,495 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:19:04,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55626 deadline: 1731208804493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:19:04,496 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:04,496 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:04,496 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:19:04,496 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:19:04,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55616 deadline: 1731208804493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:19:04,496 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:04,496 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:04,496 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:19:04,496 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:19:04,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55578 deadline: 1731208804493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:19:04,496 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:04,496 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:04,496 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:19:04,796 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:19:04,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55562 deadline: 1731208804793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:19:04,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:04,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:04,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:19:04,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:19:04,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55626 deadline: 1731208804803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:19:04,807 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:04,807 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:04,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:19:04,807 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:19:04,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55616 deadline: 1731208804803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:19:04,807 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:04,807 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:04,807 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:19:04,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:19:04,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55578 deadline: 1731208804803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:19:04,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:04,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:04,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:19:04,818 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/C/86b8f45779164e73abcb3ef7c1b466bd 2024-11-10T03:19:04,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/A/0131d2f2ee024519a36b11cbbf0c9851 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/0131d2f2ee024519a36b11cbbf0c9851 2024-11-10T03:19:04,825 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/0131d2f2ee024519a36b11cbbf0c9851, entries=150, sequenceid=263, filesize=30.5 K 2024-11-10T03:19:04,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/B/ecf1d40a3d184a25b663072129b6f03c as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/ecf1d40a3d184a25b663072129b6f03c 2024-11-10T03:19:04,830 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/ecf1d40a3d184a25b663072129b6f03c, entries=150, sequenceid=263, filesize=12.0 K 2024-11-10T03:19:04,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/C/86b8f45779164e73abcb3ef7c1b466bd as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/86b8f45779164e73abcb3ef7c1b466bd 2024-11-10T03:19:04,834 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/86b8f45779164e73abcb3ef7c1b466bd, entries=150, sequenceid=263, filesize=12.0 K 2024-11-10T03:19:04,835 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 07b306bf509fe2706c5b13e060d37e6b in 923ms, sequenceid=263, compaction requested=false 2024-11-10T03:19:04,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2603): Flush status journal for 07b306bf509fe2706c5b13e060d37e6b: 2024-11-10T03:19:04,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:19:04,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=167 2024-11-10T03:19:04,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4169): Remote procedure done, pid=167 2024-11-10T03:19:04,837 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-11-10T03:19:04,837 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0000 sec 2024-11-10T03:19:04,839 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees in 2.0050 sec 2024-11-10T03:19:04,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-10T03:19:04,983 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T03:19:04,983 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T03:19:04,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] procedure2.ProcedureExecutor(1139): Stored pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees 2024-11-10T03:19:04,985 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T03:19:04,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-10T03:19:04,985 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T03:19:04,985 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=169, ppid=168, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T03:19:05,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-10T03:19:05,137 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-10T03:19:05,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:19:05,138 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2902): Flushing 07b306bf509fe2706c5b13e060d37e6b 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-10T03:19:05,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07b306bf509fe2706c5b13e060d37e6b, store=A 2024-11-10T03:19:05,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:19:05,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07b306bf509fe2706c5b13e060d37e6b, store=B 2024-11-10T03:19:05,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:19:05,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07b306bf509fe2706c5b13e060d37e6b, store=C 2024-11-10T03:19:05,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:19:05,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411102b1251c9a545457c9b2c4190cc478943_07b306bf509fe2706c5b13e060d37e6b is 50, key is test_row_0/A:col10/1731208744167/Put/seqid=0 2024-11-10T03:19:05,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742460_1636 (size=12454) 2024-11-10T03:19:05,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-10T03:19:05,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on 07b306bf509fe2706c5b13e060d37e6b 2024-11-10T03:19:05,304 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. as already flushing 2024-11-10T03:19:05,375 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:19:05,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55626 deadline: 1731208805369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:19:05,375 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:19:05,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55616 deadline: 1731208805370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:19:05,375 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:05,375 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:05,376 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:19:05,376 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:05,376 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:05,376 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:19:05,380 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:19:05,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55562 deadline: 1731208805375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:19:05,380 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:19:05,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:05,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:05,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55578 deadline: 1731208805375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:19:05,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:19:05,381 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:05,381 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:05,381 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:19:05,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:19:05,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55626 deadline: 1731208805483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:19:05,484 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:05,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:19:05,484 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:05,484 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:19:05,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55616 deadline: 1731208805483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:19:05,484 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:05,484 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:05,484 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:19:05,489 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:19:05,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55562 deadline: 1731208805485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:19:05,489 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:05,489 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:05,489 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:19:05,496 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:19:05,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55578 deadline: 1731208805494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:19:05,497 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:05,497 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:05,497 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:19:05,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:19:05,557 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411102b1251c9a545457c9b2c4190cc478943_07b306bf509fe2706c5b13e060d37e6b to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411102b1251c9a545457c9b2c4190cc478943_07b306bf509fe2706c5b13e060d37e6b 2024-11-10T03:19:05,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/A/b69e4e9f701e461c9188f5c9af0870a7, store: [table=TestAcidGuarantees family=A region=07b306bf509fe2706c5b13e060d37e6b] 2024-11-10T03:19:05,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/A/b69e4e9f701e461c9188f5c9af0870a7 is 175, key is test_row_0/A:col10/1731208744167/Put/seqid=0 2024-11-10T03:19:05,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742461_1637 (size=31255) 2024-11-10T03:19:05,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-10T03:19:05,696 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:19:05,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55626 deadline: 1731208805692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:19:05,696 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:05,696 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:19:05,696 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:05,697 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:19:05,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55616 deadline: 1731208805693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:19:05,697 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:05,697 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:19:05,697 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:05,697 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:19:05,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55562 deadline: 1731208805693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:19:05,697 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:05,697 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:05,697 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:19:05,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:19:05,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55578 deadline: 1731208805703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:19:05,706 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:05,706 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:05,706 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:19:05,962 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=282, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/A/b69e4e9f701e461c9188f5c9af0870a7 2024-11-10T03:19:05,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/B/5e543f13bf214a2686f12ee81715b6f7 is 50, key is test_row_0/B:col10/1731208744167/Put/seqid=0 2024-11-10T03:19:05,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742462_1638 (size=12301) 2024-11-10T03:19:05,982 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/B/5e543f13bf214a2686f12ee81715b6f7 2024-11-10T03:19:05,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/C/71623c6621d04f76ba2f74216420835a is 50, key is test_row_0/C:col10/1731208744167/Put/seqid=0 2024-11-10T03:19:06,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742463_1639 (size=12301) 2024-11-10T03:19:06,006 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:19:06,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55626 deadline: 1731208806003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:19:06,006 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:06,006 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:19:06,006 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:06,006 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:19:06,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55616 deadline: 1731208806003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:19:06,006 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:06,006 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:06,006 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:19:06,007 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:19:06,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55562 deadline: 1731208806003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:19:06,007 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:06,007 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:06,007 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:19:06,017 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:19:06,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55578 deadline: 1731208806013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:19:06,018 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:06,018 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:06,018 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:19:06,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-10T03:19:06,401 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/C/71623c6621d04f76ba2f74216420835a 2024-11-10T03:19:06,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/A/b69e4e9f701e461c9188f5c9af0870a7 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/b69e4e9f701e461c9188f5c9af0870a7 2024-11-10T03:19:06,406 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/b69e4e9f701e461c9188f5c9af0870a7, entries=150, sequenceid=282, filesize=30.5 K 2024-11-10T03:19:06,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/B/5e543f13bf214a2686f12ee81715b6f7 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/5e543f13bf214a2686f12ee81715b6f7 2024-11-10T03:19:06,411 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/5e543f13bf214a2686f12ee81715b6f7, entries=150, sequenceid=282, filesize=12.0 K 2024-11-10T03:19:06,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/C/71623c6621d04f76ba2f74216420835a as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/71623c6621d04f76ba2f74216420835a 2024-11-10T03:19:06,414 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/71623c6621d04f76ba2f74216420835a, entries=150, sequenceid=282, filesize=12.0 K 2024-11-10T03:19:06,414 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 07b306bf509fe2706c5b13e060d37e6b in 1276ms, sequenceid=282, compaction requested=true 2024-11-10T03:19:06,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2603): Flush status journal for 07b306bf509fe2706c5b13e060d37e6b: 2024-11-10T03:19:06,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:19:06,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=169 2024-11-10T03:19:06,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4169): Remote procedure done, pid=169 2024-11-10T03:19:06,417 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=169, resume processing ppid=168 2024-11-10T03:19:06,417 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=169, ppid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4310 sec 2024-11-10T03:19:06,418 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees in 1.4340 sec 2024-11-10T03:19:06,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on 07b306bf509fe2706c5b13e060d37e6b 2024-11-10T03:19:06,514 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 07b306bf509fe2706c5b13e060d37e6b 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-10T03:19:06,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07b306bf509fe2706c5b13e060d37e6b, store=A 2024-11-10T03:19:06,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:19:06,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07b306bf509fe2706c5b13e060d37e6b, store=B 2024-11-10T03:19:06,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:19:06,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07b306bf509fe2706c5b13e060d37e6b, store=C 2024-11-10T03:19:06,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:19:06,520 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111033b52ebbc5da49518362d3eade86990a_07b306bf509fe2706c5b13e060d37e6b is 50, key is test_row_0/A:col10/1731208745374/Put/seqid=0 2024-11-10T03:19:06,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742464_1640 (size=14994) 2024-11-10T03:19:06,534 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:19:06,536 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111033b52ebbc5da49518362d3eade86990a_07b306bf509fe2706c5b13e060d37e6b to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111033b52ebbc5da49518362d3eade86990a_07b306bf509fe2706c5b13e060d37e6b 2024-11-10T03:19:06,537 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/A/1c699e290c9f48b1b7b0020de487ba17, store: [table=TestAcidGuarantees family=A region=07b306bf509fe2706c5b13e060d37e6b] 2024-11-10T03:19:06,538 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/A/1c699e290c9f48b1b7b0020de487ba17 is 175, key is test_row_0/A:col10/1731208745374/Put/seqid=0 2024-11-10T03:19:06,539 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:19:06,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55578 deadline: 1731208806533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:19:06,539 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:06,539 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:06,539 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:19:06,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742465_1641 (size=39949) 2024-11-10T03:19:06,542 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:19:06,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55626 deadline: 1731208806536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:19:06,542 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:06,542 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:06,542 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:19:06,542 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:19:06,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55562 deadline: 1731208806537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:19:06,543 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:06,543 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:06,543 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:19:06,543 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:19:06,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55616 deadline: 1731208806539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:19:06,544 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:06,544 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:06,544 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:19:06,654 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:19:06,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55626 deadline: 1731208806653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:19:06,655 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:06,655 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:06,655 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:19:06,658 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:19:06,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55616 deadline: 1731208806653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:19:06,659 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:19:06,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55562 deadline: 1731208806653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:19:06,659 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:06,659 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:06,659 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:19:06,659 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:06,659 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:06,659 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:19:06,865 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:19:06,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55626 deadline: 1731208806863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:19:06,866 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:19:06,866 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:06,866 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:06,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55562 deadline: 1731208806863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:19:06,866 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:19:06,866 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:06,866 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:06,866 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:19:06,867 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:19:06,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55616 deadline: 1731208806863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:19:06,867 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:06,867 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:06,867 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:19:06,956 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=299, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/A/1c699e290c9f48b1b7b0020de487ba17 2024-11-10T03:19:06,962 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/B/255a2df4142745d7bedb4a721a1e64a3 is 50, key is test_row_0/B:col10/1731208745374/Put/seqid=0 2024-11-10T03:19:06,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742466_1642 (size=12301) 2024-11-10T03:19:07,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-10T03:19:07,133 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T03:19:07,134 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T03:19:07,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] procedure2.ProcedureExecutor(1139): Stored pid=170, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees 2024-11-10T03:19:07,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-10T03:19:07,135 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=170, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T03:19:07,136 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=170, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T03:19:07,136 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=171, ppid=170, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T03:19:07,173 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:19:07,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55616 deadline: 1731208807173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:19:07,174 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:07,174 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:07,174 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:19:07,180 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:19:07,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55562 deadline: 1731208807180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:19:07,181 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:07,181 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:07,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:19:07,181 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:19:07,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55626 deadline: 1731208807180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:19:07,181 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:07,181 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:07,181 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:19:07,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-10T03:19:07,288 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-10T03:19:07,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:19:07,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. as already flushing 2024-11-10T03:19:07,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:19:07,288 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:19:07,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:19:07,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:19:07,382 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/B/255a2df4142745d7bedb4a721a1e64a3 2024-11-10T03:19:07,398 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/C/f120f452952d4a829d7c5126db8bf8ac is 50, key is test_row_0/C:col10/1731208745374/Put/seqid=0 2024-11-10T03:19:07,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742467_1643 (size=12301) 2024-11-10T03:19:07,440 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-10T03:19:07,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:19:07,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. as already flushing 2024-11-10T03:19:07,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:19:07,441 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:19:07,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:19:07,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:19:07,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-10T03:19:07,556 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:19:07,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55578 deadline: 1731208807553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:19:07,557 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 394d2222f724,38139,1731208572054 for 'test_row_0' in 07b306bf509fe2706c5b13e060d37e6b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T03:19:07,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:07,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:07,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:19:07,593 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-10T03:19:07,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:19:07,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. as already flushing 2024-11-10T03:19:07,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:19:07,593 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:19:07,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:19:07,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:19:07,686 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:19:07,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55616 deadline: 1731208807683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:19:07,687 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:07,687 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:07,687 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:19:07,708 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:19:07,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55562 deadline: 1731208807704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:19:07,708 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:19:07,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55626 deadline: 1731208807704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:19:07,708 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:07,709 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:07,709 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:07,709 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:19:07,709 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:07,709 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:19:07,749 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-10T03:19:07,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:19:07,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. as already flushing 2024-11-10T03:19:07,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:19:07,749 ERROR [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:19:07,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:19:07,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:19:07,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-10T03:19:07,802 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/C/f120f452952d4a829d7c5126db8bf8ac 2024-11-10T03:19:07,805 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/A/1c699e290c9f48b1b7b0020de487ba17 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/1c699e290c9f48b1b7b0020de487ba17 2024-11-10T03:19:07,808 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/1c699e290c9f48b1b7b0020de487ba17, entries=200, sequenceid=299, filesize=39.0 K 2024-11-10T03:19:07,808 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/B/255a2df4142745d7bedb4a721a1e64a3 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/255a2df4142745d7bedb4a721a1e64a3 2024-11-10T03:19:07,811 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/255a2df4142745d7bedb4a721a1e64a3, entries=150, sequenceid=299, filesize=12.0 K 2024-11-10T03:19:07,811 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/C/f120f452952d4a829d7c5126db8bf8ac as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/f120f452952d4a829d7c5126db8bf8ac 2024-11-10T03:19:07,813 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/f120f452952d4a829d7c5126db8bf8ac, entries=150, sequenceid=299, filesize=12.0 K 2024-11-10T03:19:07,814 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 07b306bf509fe2706c5b13e060d37e6b in 1300ms, sequenceid=299, compaction requested=true 2024-11-10T03:19:07,814 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 07b306bf509fe2706c5b13e060d37e6b: 2024-11-10T03:19:07,814 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T03:19:07,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 07b306bf509fe2706c5b13e060d37e6b:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T03:19:07,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:19:07,815 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T03:19:07,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 07b306bf509fe2706c5b13e060d37e6b:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T03:19:07,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:19:07,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 07b306bf509fe2706c5b13e060d37e6b:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T03:19:07,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T03:19:07,816 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 134178 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T03:19:07,816 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1541): 07b306bf509fe2706c5b13e060d37e6b/A is initiating minor compaction (all files) 2024-11-10T03:19:07,816 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 07b306bf509fe2706c5b13e060d37e6b/A in TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:19:07,816 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/5a5a824938bd4b5cb816103a25d4ae91, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/0131d2f2ee024519a36b11cbbf0c9851, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/b69e4e9f701e461c9188f5c9af0870a7, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/1c699e290c9f48b1b7b0020de487ba17] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp, totalSize=131.0 K 2024-11-10T03:19:07,816 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:19:07,816 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. files: [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/5a5a824938bd4b5cb816103a25d4ae91, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/0131d2f2ee024519a36b11cbbf0c9851, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/b69e4e9f701e461c9188f5c9af0870a7, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/1c699e290c9f48b1b7b0020de487ba17] 2024-11-10T03:19:07,816 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49668 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T03:19:07,816 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1541): 07b306bf509fe2706c5b13e060d37e6b/B is initiating minor compaction (all files) 2024-11-10T03:19:07,816 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 07b306bf509fe2706c5b13e060d37e6b/B in TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:19:07,816 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/ac10ffb12c06408288e9228e0fdbc7b6, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/ecf1d40a3d184a25b663072129b6f03c, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/5e543f13bf214a2686f12ee81715b6f7, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/255a2df4142745d7bedb4a721a1e64a3] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp, totalSize=48.5 K 2024-11-10T03:19:07,816 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a5a824938bd4b5cb816103a25d4ae91, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731208742341 2024-11-10T03:19:07,817 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting ac10ffb12c06408288e9228e0fdbc7b6, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731208742341 2024-11-10T03:19:07,817 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0131d2f2ee024519a36b11cbbf0c9851, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1731208742992 2024-11-10T03:19:07,817 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting ecf1d40a3d184a25b663072129b6f03c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1731208742992 2024-11-10T03:19:07,817 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting b69e4e9f701e461c9188f5c9af0870a7, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731208744163 2024-11-10T03:19:07,817 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e543f13bf214a2686f12ee81715b6f7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731208744163 2024-11-10T03:19:07,818 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c699e290c9f48b1b7b0020de487ba17, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1731208745365 2024-11-10T03:19:07,818 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 255a2df4142745d7bedb4a721a1e64a3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1731208745365 2024-11-10T03:19:07,826 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 07b306bf509fe2706c5b13e060d37e6b#B#compaction#537 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:19:07,826 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=07b306bf509fe2706c5b13e060d37e6b] 2024-11-10T03:19:07,827 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/B/9f8be0e486f5477fbbc72c5145e6a868 is 50, key is test_row_0/B:col10/1731208745374/Put/seqid=0 2024-11-10T03:19:07,829 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241110aba3c0297f2f4729915e7c9ef00bca31_07b306bf509fe2706c5b13e060d37e6b store=[table=TestAcidGuarantees family=A region=07b306bf509fe2706c5b13e060d37e6b] 2024-11-10T03:19:07,831 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241110aba3c0297f2f4729915e7c9ef00bca31_07b306bf509fe2706c5b13e060d37e6b, store=[table=TestAcidGuarantees family=A region=07b306bf509fe2706c5b13e060d37e6b] 2024-11-10T03:19:07,831 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110aba3c0297f2f4729915e7c9ef00bca31_07b306bf509fe2706c5b13e060d37e6b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=07b306bf509fe2706c5b13e060d37e6b] 2024-11-10T03:19:07,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742468_1644 (size=13051) 2024-11-10T03:19:07,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742469_1645 (size=4469) 2024-11-10T03:19:07,872 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 07b306bf509fe2706c5b13e060d37e6b#A#compaction#538 average throughput is 0.53 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:19:07,872 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/A/a1501bced7cc4fc89cb2aa4276651755 is 175, key is test_row_0/A:col10/1731208745374/Put/seqid=0 2024-11-10T03:19:07,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742470_1646 (size=32005) 2024-11-10T03:19:07,901 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38139 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-10T03:19:07,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:19:07,901 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2902): Flushing 07b306bf509fe2706c5b13e060d37e6b 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-10T03:19:07,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07b306bf509fe2706c5b13e060d37e6b, store=A 2024-11-10T03:19:07,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:19:07,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07b306bf509fe2706c5b13e060d37e6b, store=B 2024-11-10T03:19:07,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:19:07,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07b306bf509fe2706c5b13e060d37e6b, store=C 2024-11-10T03:19:07,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:19:07,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411109e546f1fd0594cb29bb04e1ebb126ffc_07b306bf509fe2706c5b13e060d37e6b is 50, key is test_row_0/A:col10/1731208746537/Put/seqid=0 2024-11-10T03:19:07,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742471_1647 (size=12454) 2024-11-10T03:19:08,253 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/B/9f8be0e486f5477fbbc72c5145e6a868 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/9f8be0e486f5477fbbc72c5145e6a868 2024-11-10T03:19:08,257 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 07b306bf509fe2706c5b13e060d37e6b/B of 07b306bf509fe2706c5b13e060d37e6b into 9f8be0e486f5477fbbc72c5145e6a868(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:19:08,257 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 07b306bf509fe2706c5b13e060d37e6b: 2024-11-10T03:19:08,257 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., storeName=07b306bf509fe2706c5b13e060d37e6b/B, priority=12, startTime=1731208747814; duration=0sec 2024-11-10T03:19:08,257 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T03:19:08,257 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 07b306bf509fe2706c5b13e060d37e6b:B 2024-11-10T03:19:08,257 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T03:19:08,258 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49668 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T03:19:08,258 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1541): 07b306bf509fe2706c5b13e060d37e6b/C is initiating minor compaction (all files) 2024-11-10T03:19:08,258 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 07b306bf509fe2706c5b13e060d37e6b/C in TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:19:08,258 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/a1a5884f6c884c26bccc421cd31011fd, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/86b8f45779164e73abcb3ef7c1b466bd, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/71623c6621d04f76ba2f74216420835a, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/f120f452952d4a829d7c5126db8bf8ac] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp, totalSize=48.5 K 2024-11-10T03:19:08,259 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting a1a5884f6c884c26bccc421cd31011fd, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731208742341 2024-11-10T03:19:08,259 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 86b8f45779164e73abcb3ef7c1b466bd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1731208742992 2024-11-10T03:19:08,259 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 71623c6621d04f76ba2f74216420835a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731208744163 2024-11-10T03:19:08,259 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting f120f452952d4a829d7c5126db8bf8ac, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1731208745365 2024-11-10T03:19:08,266 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 07b306bf509fe2706c5b13e060d37e6b#C#compaction#540 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:19:08,266 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/C/eaa6fc324897441eb6a07300409f704a is 50, key is test_row_0/C:col10/1731208745374/Put/seqid=0 2024-11-10T03:19:08,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-10T03:19:08,279 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/A/a1501bced7cc4fc89cb2aa4276651755 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/a1501bced7cc4fc89cb2aa4276651755 2024-11-10T03:19:08,282 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 07b306bf509fe2706c5b13e060d37e6b/A of 07b306bf509fe2706c5b13e060d37e6b into a1501bced7cc4fc89cb2aa4276651755(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:19:08,282 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 07b306bf509fe2706c5b13e060d37e6b: 2024-11-10T03:19:08,282 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., storeName=07b306bf509fe2706c5b13e060d37e6b/A, priority=12, startTime=1731208747814; duration=0sec 2024-11-10T03:19:08,282 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:19:08,282 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 07b306bf509fe2706c5b13e060d37e6b:A 2024-11-10T03:19:08,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742472_1648 (size=13051) 2024-11-10T03:19:08,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:19:08,317 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411109e546f1fd0594cb29bb04e1ebb126ffc_07b306bf509fe2706c5b13e060d37e6b to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411109e546f1fd0594cb29bb04e1ebb126ffc_07b306bf509fe2706c5b13e060d37e6b 2024-11-10T03:19:08,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/A/7b20ca47019b4cd09d832fdf3ef00c4a, store: [table=TestAcidGuarantees family=A region=07b306bf509fe2706c5b13e060d37e6b] 2024-11-10T03:19:08,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/A/7b20ca47019b4cd09d832fdf3ef00c4a is 175, key is test_row_0/A:col10/1731208746537/Put/seqid=0 2024-11-10T03:19:08,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742473_1649 (size=31255) 2024-11-10T03:19:08,324 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=314, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/A/7b20ca47019b4cd09d832fdf3ef00c4a 2024-11-10T03:19:08,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/B/5d8e8e9a550a434794c74a709812232e is 50, key is test_row_0/B:col10/1731208746537/Put/seqid=0 2024-11-10T03:19:08,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742474_1650 (size=12301) 2024-11-10T03:19:08,694 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/C/eaa6fc324897441eb6a07300409f704a as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/eaa6fc324897441eb6a07300409f704a 2024-11-10T03:19:08,698 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 07b306bf509fe2706c5b13e060d37e6b/C of 07b306bf509fe2706c5b13e060d37e6b into eaa6fc324897441eb6a07300409f704a(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:19:08,699 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 07b306bf509fe2706c5b13e060d37e6b: 2024-11-10T03:19:08,699 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., storeName=07b306bf509fe2706c5b13e060d37e6b/C, priority=12, startTime=1731208747815; duration=0sec 2024-11-10T03:19:08,699 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:19:08,699 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 07b306bf509fe2706c5b13e060d37e6b:C 2024-11-10T03:19:08,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on 07b306bf509fe2706c5b13e060d37e6b 2024-11-10T03:19:08,704 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. as already flushing 2024-11-10T03:19:08,733 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/B/5d8e8e9a550a434794c74a709812232e 2024-11-10T03:19:08,736 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:19:08,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55562 deadline: 1731208808732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:19:08,737 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:19:08,737 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:08,737 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:08,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55626 deadline: 1731208808732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:19:08,737 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:19:08,737 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:08,737 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:08,737 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:19:08,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/C/187590c88f2247738acbfa0f375d3be5 is 50, key is test_row_0/C:col10/1731208746537/Put/seqid=0 2024-11-10T03:19:08,740 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:19:08,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55616 deadline: 1731208808736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:19:08,740 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:08,740 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:08,740 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:19:08,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742475_1651 (size=12301) 2024-11-10T03:19:08,759 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/C/187590c88f2247738acbfa0f375d3be5 2024-11-10T03:19:08,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/A/7b20ca47019b4cd09d832fdf3ef00c4a as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/7b20ca47019b4cd09d832fdf3ef00c4a 2024-11-10T03:19:08,768 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/7b20ca47019b4cd09d832fdf3ef00c4a, entries=150, sequenceid=314, filesize=30.5 K 2024-11-10T03:19:08,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/B/5d8e8e9a550a434794c74a709812232e as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/5d8e8e9a550a434794c74a709812232e 2024-11-10T03:19:08,771 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/5d8e8e9a550a434794c74a709812232e, entries=150, sequenceid=314, filesize=12.0 K 2024-11-10T03:19:08,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/C/187590c88f2247738acbfa0f375d3be5 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/187590c88f2247738acbfa0f375d3be5 2024-11-10T03:19:08,774 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/187590c88f2247738acbfa0f375d3be5, entries=150, sequenceid=314, filesize=12.0 K 2024-11-10T03:19:08,775 INFO [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 07b306bf509fe2706c5b13e060d37e6b in 874ms, sequenceid=314, compaction requested=false 2024-11-10T03:19:08,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2603): Flush status journal for 07b306bf509fe2706c5b13e060d37e6b: 2024-11-10T03:19:08,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:19:08,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/394d2222f724:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=171 2024-11-10T03:19:08,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster(4169): Remote procedure done, pid=171 2024-11-10T03:19:08,778 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=171, resume processing ppid=170 2024-11-10T03:19:08,778 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=171, ppid=170, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6400 sec 2024-11-10T03:19:08,782 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=170, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees in 1.6440 sec 2024-11-10T03:19:08,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on 07b306bf509fe2706c5b13e060d37e6b 2024-11-10T03:19:08,845 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 07b306bf509fe2706c5b13e060d37e6b 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-10T03:19:08,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07b306bf509fe2706c5b13e060d37e6b, store=A 2024-11-10T03:19:08,846 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:19:08,846 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07b306bf509fe2706c5b13e060d37e6b, store=B 2024-11-10T03:19:08,846 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:19:08,846 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07b306bf509fe2706c5b13e060d37e6b, store=C 2024-11-10T03:19:08,846 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:19:08,851 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110d8d7029b0d984bb790764720c6457e17_07b306bf509fe2706c5b13e060d37e6b is 50, key is test_row_0/A:col10/1731208748734/Put/seqid=0 2024-11-10T03:19:08,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742476_1652 (size=12454) 2024-11-10T03:19:08,855 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:19:08,857 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110d8d7029b0d984bb790764720c6457e17_07b306bf509fe2706c5b13e060d37e6b to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110d8d7029b0d984bb790764720c6457e17_07b306bf509fe2706c5b13e060d37e6b 2024-11-10T03:19:08,858 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/A/a47b00f6ff5a4454af2ae9e0688f0316, store: [table=TestAcidGuarantees family=A region=07b306bf509fe2706c5b13e060d37e6b] 2024-11-10T03:19:08,858 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/A/a47b00f6ff5a4454af2ae9e0688f0316 is 175, key is test_row_0/A:col10/1731208748734/Put/seqid=0 2024-11-10T03:19:08,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742477_1653 (size=31255) 2024-11-10T03:19:08,861 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=335, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/A/a47b00f6ff5a4454af2ae9e0688f0316 2024-11-10T03:19:08,866 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:19:08,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55626 deadline: 1731208808861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:19:08,867 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:08,867 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:08,867 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:19:08,867 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:19:08,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55616 deadline: 1731208808861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:19:08,867 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:19:08,867 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:08,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55562 deadline: 1731208808862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:19:08,867 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:08,867 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:19:08,868 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:08,868 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:08,868 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/B/2f23dd9091c54eb79f3a67340b32f0dc is 50, key is test_row_0/B:col10/1731208748734/Put/seqid=0 2024-11-10T03:19:08,868 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:19:08,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742478_1654 (size=12301) 2024-11-10T03:19:08,921 INFO [Thread-2484 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2484. 2024-11-10T03:19:08,921 DEBUG [Thread-2484 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T03:19:08,921 DEBUG [Thread-2484 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:19:08,922 DEBUG [Thread-2484 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:19:08,922 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T03:19:08,922 INFO [Thread-2478 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2478. 2024-11-10T03:19:08,922 DEBUG [Thread-2478 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T03:19:08,922 DEBUG [Thread-2478 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:19:08,922 DEBUG [Thread-2478 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:19:08,922 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T03:19:08,925 INFO [Thread-2482 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2482. 2024-11-10T03:19:08,925 DEBUG [Thread-2482 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T03:19:08,925 DEBUG [Thread-2482 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:19:08,925 DEBUG [Thread-2482 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:19:08,925 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T03:19:08,925 INFO [Thread-2480 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2480. 2024-11-10T03:19:08,925 DEBUG [Thread-2480 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T03:19:08,925 DEBUG [Thread-2480 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:19:08,925 DEBUG [Thread-2480 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:19:08,925 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T03:19:08,926 INFO [Thread-2486 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2486. 2024-11-10T03:19:08,926 DEBUG [Thread-2486 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T03:19:08,926 DEBUG [Thread-2486 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:19:08,927 DEBUG [Thread-2486 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:19:08,927 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T03:19:08,973 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:19:08,973 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:19:08,974 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:19:08,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55616 deadline: 1731208808973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:19:08,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55562 deadline: 1731208808973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:19:08,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55626 deadline: 1731208808973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:19:08,974 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:08,974 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:08,974 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:08,974 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:08,974 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:08,974 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:08,974 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:19:08,974 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:19:08,974 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:19:09,183 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:19:09,183 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:19:09,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:19:09,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55562 deadline: 1731208809183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:19:09,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55626 deadline: 1731208809183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:19:09,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55616 deadline: 1731208809183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:19:09,184 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:09,184 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:09,184 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:09,184 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:09,184 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:09,184 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:09,184 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:19:09,184 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:19:09,184 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:19:09,271 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/B/2f23dd9091c54eb79f3a67340b32f0dc 2024-11-10T03:19:09,277 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/C/93eacede815c49649faa2934879ce7d2 is 50, key is test_row_0/C:col10/1731208748734/Put/seqid=0 2024-11-10T03:19:09,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742479_1655 (size=12301) 2024-11-10T03:19:09,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-10T03:19:09,283 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T03:19:09,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:19:09,493 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:19:09,493 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:19:09,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55616 deadline: 1731208809493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:19:09,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55562 deadline: 1731208809493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:19:09,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55626 deadline: 1731208809493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:19:09,494 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:09,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:09,494 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:09,494 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:09,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:09,494 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:09,494 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:19:09,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:19:09,494 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:19:09,573 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:19:09,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38139 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55578 deadline: 1731208809573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 2024-11-10T03:19:09,574 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 394d2222f724,38139,1731208572054 for 'test_row_0' in 07b306bf509fe2706c5b13e060d37e6b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4198 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T03:19:09,574 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 , the old value is region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:09,574 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T03:19:09,574 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., hostname=394d2222f724,38139,1731208572054, seqNum=5 because the exception is null or not the one we care about 2024-11-10T03:19:09,680 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/C/93eacede815c49649faa2934879ce7d2 2024-11-10T03:19:09,683 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/A/a47b00f6ff5a4454af2ae9e0688f0316 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/a47b00f6ff5a4454af2ae9e0688f0316 2024-11-10T03:19:09,686 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/a47b00f6ff5a4454af2ae9e0688f0316, entries=150, sequenceid=335, filesize=30.5 K 2024-11-10T03:19:09,686 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/B/2f23dd9091c54eb79f3a67340b32f0dc as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/2f23dd9091c54eb79f3a67340b32f0dc 2024-11-10T03:19:09,689 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/2f23dd9091c54eb79f3a67340b32f0dc, entries=150, sequenceid=335, filesize=12.0 K 2024-11-10T03:19:09,689 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/C/93eacede815c49649faa2934879ce7d2 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/93eacede815c49649faa2934879ce7d2 2024-11-10T03:19:09,691 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/93eacede815c49649faa2934879ce7d2, entries=150, sequenceid=335, filesize=12.0 K 2024-11-10T03:19:09,692 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 07b306bf509fe2706c5b13e060d37e6b in 847ms, sequenceid=335, compaction requested=true 2024-11-10T03:19:09,692 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 07b306bf509fe2706c5b13e060d37e6b: 2024-11-10T03:19:09,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 07b306bf509fe2706c5b13e060d37e6b:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T03:19:09,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:19:09,692 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T03:19:09,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 07b306bf509fe2706c5b13e060d37e6b:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T03:19:09,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:19:09,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 07b306bf509fe2706c5b13e060d37e6b:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T03:19:09,692 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T03:19:09,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T03:19:09,693 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94515 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T03:19:09,693 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T03:19:09,693 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1541): 07b306bf509fe2706c5b13e060d37e6b/B is initiating minor compaction (all files) 2024-11-10T03:19:09,693 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1541): 07b306bf509fe2706c5b13e060d37e6b/A is initiating minor compaction (all files) 2024-11-10T03:19:09,693 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 07b306bf509fe2706c5b13e060d37e6b/A in TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:19:09,693 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 07b306bf509fe2706c5b13e060d37e6b/B in TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:19:09,693 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/a1501bced7cc4fc89cb2aa4276651755, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/7b20ca47019b4cd09d832fdf3ef00c4a, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/a47b00f6ff5a4454af2ae9e0688f0316] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp, totalSize=92.3 K 2024-11-10T03:19:09,693 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/9f8be0e486f5477fbbc72c5145e6a868, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/5d8e8e9a550a434794c74a709812232e, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/2f23dd9091c54eb79f3a67340b32f0dc] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp, totalSize=36.8 K 2024-11-10T03:19:09,693 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:19:09,693 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. files: [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/a1501bced7cc4fc89cb2aa4276651755, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/7b20ca47019b4cd09d832fdf3ef00c4a, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/a47b00f6ff5a4454af2ae9e0688f0316] 2024-11-10T03:19:09,693 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f8be0e486f5477fbbc72c5145e6a868, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1731208745365 2024-11-10T03:19:09,693 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting a1501bced7cc4fc89cb2aa4276651755, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1731208745365 2024-11-10T03:19:09,693 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d8e8e9a550a434794c74a709812232e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1731208746519 2024-11-10T03:19:09,693 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b20ca47019b4cd09d832fdf3ef00c4a, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1731208746519 2024-11-10T03:19:09,694 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f23dd9091c54eb79f3a67340b32f0dc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1731208748731 2024-11-10T03:19:09,694 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] compactions.Compactor(225): Compacting a47b00f6ff5a4454af2ae9e0688f0316, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1731208748731 2024-11-10T03:19:09,697 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=07b306bf509fe2706c5b13e060d37e6b] 2024-11-10T03:19:09,698 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 07b306bf509fe2706c5b13e060d37e6b#B#compaction#546 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:19:09,698 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/B/4cd9e2afaaa7455499b908fb0cd0bc32 is 50, key is test_row_0/B:col10/1731208748734/Put/seqid=0 2024-11-10T03:19:09,699 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241110aaf4ba4ce3874209aa3a9df125d437ad_07b306bf509fe2706c5b13e060d37e6b store=[table=TestAcidGuarantees family=A region=07b306bf509fe2706c5b13e060d37e6b] 2024-11-10T03:19:09,702 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241110aaf4ba4ce3874209aa3a9df125d437ad_07b306bf509fe2706c5b13e060d37e6b, store=[table=TestAcidGuarantees family=A region=07b306bf509fe2706c5b13e060d37e6b] 2024-11-10T03:19:09,702 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110aaf4ba4ce3874209aa3a9df125d437ad_07b306bf509fe2706c5b13e060d37e6b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=07b306bf509fe2706c5b13e060d37e6b] 2024-11-10T03:19:09,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742480_1656 (size=13153) 2024-11-10T03:19:09,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742481_1657 (size=4469) 2024-11-10T03:19:10,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38139 {}] regionserver.HRegion(8855): Flush requested on 07b306bf509fe2706c5b13e060d37e6b 2024-11-10T03:19:10,004 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 07b306bf509fe2706c5b13e060d37e6b 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-10T03:19:10,004 INFO [Thread-2467 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2467. 2024-11-10T03:19:10,004 DEBUG [Thread-2467 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T03:19:10,004 DEBUG [Thread-2467 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:19:10,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07b306bf509fe2706c5b13e060d37e6b, store=A 2024-11-10T03:19:10,004 INFO [Thread-2475 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2475. 2024-11-10T03:19:10,004 INFO [Thread-2469 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2469. 2024-11-10T03:19:10,004 DEBUG [Thread-2467 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:19:10,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:19:10,004 DEBUG [Thread-2475 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T03:19:10,004 DEBUG [Thread-2469 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T03:19:10,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07b306bf509fe2706c5b13e060d37e6b, store=B 2024-11-10T03:19:10,004 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T03:19:10,004 DEBUG [Thread-2475 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:19:10,004 DEBUG [Thread-2469 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:19:10,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:19:10,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07b306bf509fe2706c5b13e060d37e6b, store=C 2024-11-10T03:19:10,004 DEBUG [Thread-2475 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:19:10,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:19:10,004 DEBUG [Thread-2469 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:19:10,004 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T03:19:10,004 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T03:19:10,009 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111070757c4084e744c68cea3b759f33d9c7_07b306bf509fe2706c5b13e060d37e6b is 50, key is test_row_0/A:col10/1731208748860/Put/seqid=0 2024-11-10T03:19:10,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742482_1658 (size=12454) 2024-11-10T03:19:10,106 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 07b306bf509fe2706c5b13e060d37e6b#A#compaction#547 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:19:10,107 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/A/3f20212038bd439c8798af36a0c9a1ba is 175, key is test_row_0/A:col10/1731208748734/Put/seqid=0 2024-11-10T03:19:10,107 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/B/4cd9e2afaaa7455499b908fb0cd0bc32 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/4cd9e2afaaa7455499b908fb0cd0bc32 2024-11-10T03:19:10,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742483_1659 (size=32107) 2024-11-10T03:19:10,110 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 07b306bf509fe2706c5b13e060d37e6b/B of 07b306bf509fe2706c5b13e060d37e6b into 4cd9e2afaaa7455499b908fb0cd0bc32(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:19:10,111 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 07b306bf509fe2706c5b13e060d37e6b: 2024-11-10T03:19:10,111 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., storeName=07b306bf509fe2706c5b13e060d37e6b/B, priority=13, startTime=1731208749692; duration=0sec 2024-11-10T03:19:10,111 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T03:19:10,111 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 07b306bf509fe2706c5b13e060d37e6b:B 2024-11-10T03:19:10,111 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T03:19:10,112 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T03:19:10,112 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1541): 07b306bf509fe2706c5b13e060d37e6b/C is initiating minor compaction (all files) 2024-11-10T03:19:10,112 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 07b306bf509fe2706c5b13e060d37e6b/C in TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:19:10,112 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/eaa6fc324897441eb6a07300409f704a, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/187590c88f2247738acbfa0f375d3be5, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/93eacede815c49649faa2934879ce7d2] into tmpdir=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp, totalSize=36.8 K 2024-11-10T03:19:10,112 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting eaa6fc324897441eb6a07300409f704a, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1731208745365 2024-11-10T03:19:10,112 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 187590c88f2247738acbfa0f375d3be5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1731208746519 2024-11-10T03:19:10,113 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] compactions.Compactor(225): Compacting 93eacede815c49649faa2934879ce7d2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1731208748731 2024-11-10T03:19:10,118 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 07b306bf509fe2706c5b13e060d37e6b#C#compaction#549 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T03:19:10,118 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/C/84cc219363104a399ee5a4fd8c28f516 is 50, key is test_row_0/C:col10/1731208748734/Put/seqid=0 2024-11-10T03:19:10,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742484_1660 (size=13153) 2024-11-10T03:19:10,312 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-10T03:19:10,412 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:19:10,415 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111070757c4084e744c68cea3b759f33d9c7_07b306bf509fe2706c5b13e060d37e6b to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111070757c4084e744c68cea3b759f33d9c7_07b306bf509fe2706c5b13e060d37e6b 2024-11-10T03:19:10,416 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/A/9bbd33d092a444c7b5c63e26ea84a5d9, store: [table=TestAcidGuarantees family=A region=07b306bf509fe2706c5b13e060d37e6b] 2024-11-10T03:19:10,417 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/A/9bbd33d092a444c7b5c63e26ea84a5d9 is 175, key is test_row_0/A:col10/1731208748860/Put/seqid=0 2024-11-10T03:19:10,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742485_1661 (size=31255) 2024-11-10T03:19:10,513 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/A/3f20212038bd439c8798af36a0c9a1ba as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/3f20212038bd439c8798af36a0c9a1ba 2024-11-10T03:19:10,517 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 07b306bf509fe2706c5b13e060d37e6b/A of 07b306bf509fe2706c5b13e060d37e6b into 3f20212038bd439c8798af36a0c9a1ba(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:19:10,517 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 07b306bf509fe2706c5b13e060d37e6b: 2024-11-10T03:19:10,517 INFO [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., storeName=07b306bf509fe2706c5b13e060d37e6b/A, priority=13, startTime=1731208749692; duration=0sec 2024-11-10T03:19:10,517 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:19:10,517 DEBUG [RS:0;394d2222f724:38139-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 07b306bf509fe2706c5b13e060d37e6b:A 2024-11-10T03:19:10,524 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/C/84cc219363104a399ee5a4fd8c28f516 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/84cc219363104a399ee5a4fd8c28f516 2024-11-10T03:19:10,528 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 07b306bf509fe2706c5b13e060d37e6b/C of 07b306bf509fe2706c5b13e060d37e6b into 84cc219363104a399ee5a4fd8c28f516(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T03:19:10,528 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 07b306bf509fe2706c5b13e060d37e6b: 2024-11-10T03:19:10,528 INFO [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., storeName=07b306bf509fe2706c5b13e060d37e6b/C, priority=13, startTime=1731208749692; duration=0sec 2024-11-10T03:19:10,528 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T03:19:10,528 DEBUG [RS:0;394d2222f724:38139-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 07b306bf509fe2706c5b13e060d37e6b:C 2024-11-10T03:19:10,820 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=352, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/A/9bbd33d092a444c7b5c63e26ea84a5d9 2024-11-10T03:19:10,826 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/B/755ecccb055a4c72af363ee6ebd33aff is 50, key is test_row_0/B:col10/1731208748860/Put/seqid=0 2024-11-10T03:19:10,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742486_1662 (size=12301) 2024-11-10T03:19:11,229 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/B/755ecccb055a4c72af363ee6ebd33aff 2024-11-10T03:19:11,235 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/C/d172de31786241c5b3ada95ddc783c7a is 50, key is test_row_0/C:col10/1731208748860/Put/seqid=0 2024-11-10T03:19:11,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742487_1663 (size=12301) 2024-11-10T03:19:11,641 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/C/d172de31786241c5b3ada95ddc783c7a 2024-11-10T03:19:11,644 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/A/9bbd33d092a444c7b5c63e26ea84a5d9 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/9bbd33d092a444c7b5c63e26ea84a5d9 2024-11-10T03:19:11,646 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/9bbd33d092a444c7b5c63e26ea84a5d9, entries=150, sequenceid=352, filesize=30.5 K 2024-11-10T03:19:11,647 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/B/755ecccb055a4c72af363ee6ebd33aff as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/755ecccb055a4c72af363ee6ebd33aff 2024-11-10T03:19:11,649 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/755ecccb055a4c72af363ee6ebd33aff, entries=150, sequenceid=352, filesize=12.0 K 2024-11-10T03:19:11,650 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/C/d172de31786241c5b3ada95ddc783c7a as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/d172de31786241c5b3ada95ddc783c7a 2024-11-10T03:19:11,652 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/d172de31786241c5b3ada95ddc783c7a, entries=150, sequenceid=352, filesize=12.0 K 2024-11-10T03:19:11,652 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=0 B/0 for 07b306bf509fe2706c5b13e060d37e6b in 1648ms, sequenceid=352, compaction requested=false 2024-11-10T03:19:11,653 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 07b306bf509fe2706c5b13e060d37e6b: 2024-11-10T03:19:13,504 INFO [Thread-2471 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2471. 2024-11-10T03:19:13,504 DEBUG [Thread-2471 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T03:19:13,504 DEBUG [Thread-2471 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:19:13,504 DEBUG [Thread-2471 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:19:13,504 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T03:19:13,614 INFO [Thread-2473 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2473. 2024-11-10T03:19:13,614 DEBUG [Thread-2473 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T03:19:13,614 DEBUG [Thread-2473 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:19:13,614 DEBUG [Thread-2473 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:19:13,614 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T03:19:13,614 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-10T03:19:13,614 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 62 2024-11-10T03:19:13,614 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 62 2024-11-10T03:19:13,614 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 39 2024-11-10T03:19:13,614 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 45 2024-11-10T03:19:13,614 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 64 2024-11-10T03:19:13,614 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-10T03:19:13,614 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2991 2024-11-10T03:19:13,614 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3099 2024-11-10T03:19:13,615 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3014 2024-11-10T03:19:13,615 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2993 2024-11-10T03:19:13,615 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3093 2024-11-10T03:19:13,615 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-10T03:19:13,615 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T03:19:13,615 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T03:19:13,615 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:19:13,615 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:19:13,615 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T03:19:13,616 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-10T03:19:13,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] procedure2.ProcedureExecutor(1139): Stored pid=172, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-10T03:19:13,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-10T03:19:13,618 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731208753618"}]},"ts":"1731208753618"} 2024-11-10T03:19:13,619 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-10T03:19:13,619 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-10T03:19:13,620 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=173, ppid=172, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-10T03:19:13,621 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=174, ppid=173, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=07b306bf509fe2706c5b13e060d37e6b, UNASSIGN}] 2024-11-10T03:19:13,622 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=174, ppid=173, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=07b306bf509fe2706c5b13e060d37e6b, UNASSIGN 2024-11-10T03:19:13,623 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=174 updating hbase:meta row=07b306bf509fe2706c5b13e060d37e6b, regionState=CLOSING, regionLocation=394d2222f724,38139,1731208572054 2024-11-10T03:19:13,624 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=174, ppid=173, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=07b306bf509fe2706c5b13e060d37e6b, UNASSIGN because future has completed 2024-11-10T03:19:13,624 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-10T03:19:13,624 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=175, ppid=174, state=RUNNABLE, hasLock=false; CloseRegionProcedure 07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054}] 2024-11-10T03:19:13,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-10T03:19:13,776 INFO [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] handler.UnassignRegionHandler(122): Close 07b306bf509fe2706c5b13e060d37e6b 2024-11-10T03:19:13,776 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-10T03:19:13,776 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1722): Closing 07b306bf509fe2706c5b13e060d37e6b, disabling compactions & flushes 2024-11-10T03:19:13,777 INFO [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:19:13,777 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:19:13,777 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. after waiting 0 ms 2024-11-10T03:19:13,777 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:19:13,777 INFO [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(2902): Flushing 07b306bf509fe2706c5b13e060d37e6b 3/3 column families, dataSize=13.42 KB heapSize=40.59 KB 2024-11-10T03:19:13,777 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07b306bf509fe2706c5b13e060d37e6b, store=A 2024-11-10T03:19:13,777 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:19:13,777 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07b306bf509fe2706c5b13e060d37e6b, store=B 2024-11-10T03:19:13,777 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:19:13,777 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07b306bf509fe2706c5b13e060d37e6b, store=C 2024-11-10T03:19:13,777 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T03:19:13,781 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110c37255c2b6eb4ae9bbd980f28c21def1_07b306bf509fe2706c5b13e060d37e6b is 50, key is test_row_0/A:col10/1731208753613/Put/seqid=0 2024-11-10T03:19:13,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742488_1664 (size=9914) 2024-11-10T03:19:13,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-10T03:19:14,184 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:19:14,187 INFO [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110c37255c2b6eb4ae9bbd980f28c21def1_07b306bf509fe2706c5b13e060d37e6b to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110c37255c2b6eb4ae9bbd980f28c21def1_07b306bf509fe2706c5b13e060d37e6b 2024-11-10T03:19:14,188 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/A/47e6bbda94064bc696411c4325871786, store: [table=TestAcidGuarantees family=A region=07b306bf509fe2706c5b13e060d37e6b] 2024-11-10T03:19:14,189 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/A/47e6bbda94064bc696411c4325871786 is 175, key is test_row_0/A:col10/1731208753613/Put/seqid=0 2024-11-10T03:19:14,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742489_1665 (size=22561) 2024-11-10T03:19:14,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-10T03:19:14,592 INFO [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=360, memsize=4.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/A/47e6bbda94064bc696411c4325871786 2024-11-10T03:19:14,597 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/B/dbf735c0f71943dcb3c9c2c20eb62efb is 50, key is test_row_0/B:col10/1731208753613/Put/seqid=0 2024-11-10T03:19:14,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742490_1666 (size=9857) 2024-11-10T03:19:14,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-10T03:19:15,000 INFO [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/B/dbf735c0f71943dcb3c9c2c20eb62efb 2024-11-10T03:19:15,005 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/C/61a4c4eea80c4632aeccf688e7610f95 is 50, key is test_row_0/C:col10/1731208753613/Put/seqid=0 2024-11-10T03:19:15,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742491_1667 (size=9857) 2024-11-10T03:19:15,409 INFO [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/C/61a4c4eea80c4632aeccf688e7610f95 2024-11-10T03:19:15,412 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/A/47e6bbda94064bc696411c4325871786 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/47e6bbda94064bc696411c4325871786 2024-11-10T03:19:15,414 INFO [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/47e6bbda94064bc696411c4325871786, entries=100, sequenceid=360, filesize=22.0 K 2024-11-10T03:19:15,415 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/B/dbf735c0f71943dcb3c9c2c20eb62efb as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/dbf735c0f71943dcb3c9c2c20eb62efb 2024-11-10T03:19:15,417 INFO [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/dbf735c0f71943dcb3c9c2c20eb62efb, entries=100, sequenceid=360, filesize=9.6 K 2024-11-10T03:19:15,418 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/.tmp/C/61a4c4eea80c4632aeccf688e7610f95 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/61a4c4eea80c4632aeccf688e7610f95 2024-11-10T03:19:15,420 INFO [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/61a4c4eea80c4632aeccf688e7610f95, entries=100, sequenceid=360, filesize=9.6 K 2024-11-10T03:19:15,421 INFO [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 07b306bf509fe2706c5b13e060d37e6b in 1643ms, sequenceid=360, compaction requested=true 2024-11-10T03:19:15,421 DEBUG [StoreCloser-TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/b1d23fd2280d4ae8a723eb890c968779, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/0750e0f42c38424b83cbe7d376492cf3, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/5b33ff44395d4a16b2e28d68edf8d8c3, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/86c672fa6e1b452493e686dc87a872aa, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/f06bf60d296046a9b4e5418f2854966f, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/a723af900541489fbdd25c3e8fe6675f, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/757c0046f47e4aa0baba7c72e9dc8762, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/eb197861ff5744c8a1b86e27afe577de, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/f256e8b29d5c43148f01d7fc82b35581, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/685288664520438eb094b998765b5264, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/e9322848bab7494cbc8d7e60323f53ba, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/c146dc10e17441a58ee3d9a5655cdf4f, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/264b78c89cbf45d499500b81ce3304a2, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/f9ba9aed7ae749509794b7b2dc2c1554, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/d9a3a63a766d4a7486b5df08d3574d93, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/860047d56dde470c8b0af3cc502ac655, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/3afb1fe1836c476f8741dea4412bbae5, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/259bfaf7b8ce43a6a90507168277f930, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/5a5a824938bd4b5cb816103a25d4ae91, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/0131d2f2ee024519a36b11cbbf0c9851, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/b69e4e9f701e461c9188f5c9af0870a7, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/1c699e290c9f48b1b7b0020de487ba17, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/a1501bced7cc4fc89cb2aa4276651755, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/7b20ca47019b4cd09d832fdf3ef00c4a, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/a47b00f6ff5a4454af2ae9e0688f0316] to archive 2024-11-10T03:19:15,422 DEBUG [StoreCloser-TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T03:19:15,423 DEBUG [StoreCloser-TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/b1d23fd2280d4ae8a723eb890c968779 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/b1d23fd2280d4ae8a723eb890c968779 2024-11-10T03:19:15,424 DEBUG [StoreCloser-TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/0750e0f42c38424b83cbe7d376492cf3 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/0750e0f42c38424b83cbe7d376492cf3 2024-11-10T03:19:15,425 DEBUG [StoreCloser-TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/5b33ff44395d4a16b2e28d68edf8d8c3 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/5b33ff44395d4a16b2e28d68edf8d8c3 2024-11-10T03:19:15,426 DEBUG [StoreCloser-TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/86c672fa6e1b452493e686dc87a872aa to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/86c672fa6e1b452493e686dc87a872aa 2024-11-10T03:19:15,426 DEBUG [StoreCloser-TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/f06bf60d296046a9b4e5418f2854966f to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/f06bf60d296046a9b4e5418f2854966f 2024-11-10T03:19:15,427 DEBUG [StoreCloser-TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/a723af900541489fbdd25c3e8fe6675f to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/a723af900541489fbdd25c3e8fe6675f 2024-11-10T03:19:15,428 DEBUG [StoreCloser-TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/757c0046f47e4aa0baba7c72e9dc8762 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/757c0046f47e4aa0baba7c72e9dc8762 2024-11-10T03:19:15,428 DEBUG [StoreCloser-TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/eb197861ff5744c8a1b86e27afe577de to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/eb197861ff5744c8a1b86e27afe577de 2024-11-10T03:19:15,429 DEBUG [StoreCloser-TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/f256e8b29d5c43148f01d7fc82b35581 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/f256e8b29d5c43148f01d7fc82b35581 2024-11-10T03:19:15,430 DEBUG [StoreCloser-TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/685288664520438eb094b998765b5264 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/685288664520438eb094b998765b5264 2024-11-10T03:19:15,431 DEBUG [StoreCloser-TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/e9322848bab7494cbc8d7e60323f53ba to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/e9322848bab7494cbc8d7e60323f53ba 2024-11-10T03:19:15,431 DEBUG [StoreCloser-TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/c146dc10e17441a58ee3d9a5655cdf4f to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/c146dc10e17441a58ee3d9a5655cdf4f 2024-11-10T03:19:15,432 DEBUG [StoreCloser-TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/264b78c89cbf45d499500b81ce3304a2 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/264b78c89cbf45d499500b81ce3304a2 2024-11-10T03:19:15,432 DEBUG [StoreCloser-TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/f9ba9aed7ae749509794b7b2dc2c1554 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/f9ba9aed7ae749509794b7b2dc2c1554 2024-11-10T03:19:15,433 DEBUG [StoreCloser-TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/d9a3a63a766d4a7486b5df08d3574d93 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/d9a3a63a766d4a7486b5df08d3574d93 2024-11-10T03:19:15,434 DEBUG [StoreCloser-TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/860047d56dde470c8b0af3cc502ac655 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/860047d56dde470c8b0af3cc502ac655 2024-11-10T03:19:15,435 DEBUG [StoreCloser-TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/3afb1fe1836c476f8741dea4412bbae5 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/3afb1fe1836c476f8741dea4412bbae5 2024-11-10T03:19:15,435 DEBUG [StoreCloser-TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/259bfaf7b8ce43a6a90507168277f930 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/259bfaf7b8ce43a6a90507168277f930 2024-11-10T03:19:15,436 DEBUG [StoreCloser-TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/5a5a824938bd4b5cb816103a25d4ae91 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/5a5a824938bd4b5cb816103a25d4ae91 2024-11-10T03:19:15,437 DEBUG [StoreCloser-TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/0131d2f2ee024519a36b11cbbf0c9851 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/0131d2f2ee024519a36b11cbbf0c9851 2024-11-10T03:19:15,437 DEBUG [StoreCloser-TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/b69e4e9f701e461c9188f5c9af0870a7 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/b69e4e9f701e461c9188f5c9af0870a7 2024-11-10T03:19:15,438 DEBUG [StoreCloser-TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/1c699e290c9f48b1b7b0020de487ba17 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/1c699e290c9f48b1b7b0020de487ba17 2024-11-10T03:19:15,439 DEBUG [StoreCloser-TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/a1501bced7cc4fc89cb2aa4276651755 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/a1501bced7cc4fc89cb2aa4276651755 2024-11-10T03:19:15,439 DEBUG [StoreCloser-TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/7b20ca47019b4cd09d832fdf3ef00c4a to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/7b20ca47019b4cd09d832fdf3ef00c4a 2024-11-10T03:19:15,440 DEBUG [StoreCloser-TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/a47b00f6ff5a4454af2ae9e0688f0316 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/a47b00f6ff5a4454af2ae9e0688f0316 2024-11-10T03:19:15,441 DEBUG [StoreCloser-TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/34beb169150f42499b6fb0520d48d360, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/7cad940719884d5fb0e686c473e0ab5f, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/6083849e7b91410bbbabfda04b979196, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/2aa06ccc9b244d3a925ab45f8b2fdff1, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/49ac62fb2fac4c7da76e3cec5f269287, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/6a48249176dd453bb6ac913226d066e2, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/6ce48c361c694785b0c7603472751c44, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/1710ac918c814b55a9cedf78c4f49d69, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/fafa0676e1944bb5a4886af093d6f596, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/1517f097ce6746a5bdbd03f3e535717c, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/49a3aa1e4c82453f856b12109cc877dd, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/4defd168fa114b55800e15bc259371f0, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/20eaa08314c845b0a434b7427807a0ac, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/c4970472e33446819b39dbf2662298a5, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/abc3e04fcd244982880cda8c79b4cb7d, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/c377e74ec4ce4f53b86de58c3056d232, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/9bd9781a72784d23bb675e83a9b4f1ba, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/ac10ffb12c06408288e9228e0fdbc7b6, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/cd5f885188244e999b4e6c5a7ee5a1cb, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/ecf1d40a3d184a25b663072129b6f03c, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/5e543f13bf214a2686f12ee81715b6f7, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/9f8be0e486f5477fbbc72c5145e6a868, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/255a2df4142745d7bedb4a721a1e64a3, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/5d8e8e9a550a434794c74a709812232e, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/2f23dd9091c54eb79f3a67340b32f0dc] to archive 2024-11-10T03:19:15,441 DEBUG [StoreCloser-TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T03:19:15,442 DEBUG [StoreCloser-TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/34beb169150f42499b6fb0520d48d360 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/34beb169150f42499b6fb0520d48d360 2024-11-10T03:19:15,443 DEBUG [StoreCloser-TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/7cad940719884d5fb0e686c473e0ab5f to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/7cad940719884d5fb0e686c473e0ab5f 2024-11-10T03:19:15,444 DEBUG [StoreCloser-TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/6083849e7b91410bbbabfda04b979196 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/6083849e7b91410bbbabfda04b979196 2024-11-10T03:19:15,445 DEBUG [StoreCloser-TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/2aa06ccc9b244d3a925ab45f8b2fdff1 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/2aa06ccc9b244d3a925ab45f8b2fdff1 2024-11-10T03:19:15,445 DEBUG [StoreCloser-TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/49ac62fb2fac4c7da76e3cec5f269287 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/49ac62fb2fac4c7da76e3cec5f269287 2024-11-10T03:19:15,446 DEBUG [StoreCloser-TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/6a48249176dd453bb6ac913226d066e2 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/6a48249176dd453bb6ac913226d066e2 2024-11-10T03:19:15,446 DEBUG [StoreCloser-TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/6ce48c361c694785b0c7603472751c44 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/6ce48c361c694785b0c7603472751c44 2024-11-10T03:19:15,447 DEBUG [StoreCloser-TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/1710ac918c814b55a9cedf78c4f49d69 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/1710ac918c814b55a9cedf78c4f49d69 2024-11-10T03:19:15,448 DEBUG [StoreCloser-TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/fafa0676e1944bb5a4886af093d6f596 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/fafa0676e1944bb5a4886af093d6f596 2024-11-10T03:19:15,448 DEBUG [StoreCloser-TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/1517f097ce6746a5bdbd03f3e535717c to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/1517f097ce6746a5bdbd03f3e535717c 2024-11-10T03:19:15,449 DEBUG [StoreCloser-TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/49a3aa1e4c82453f856b12109cc877dd to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/49a3aa1e4c82453f856b12109cc877dd 2024-11-10T03:19:15,450 DEBUG [StoreCloser-TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/4defd168fa114b55800e15bc259371f0 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/4defd168fa114b55800e15bc259371f0 2024-11-10T03:19:15,450 DEBUG [StoreCloser-TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/20eaa08314c845b0a434b7427807a0ac to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/20eaa08314c845b0a434b7427807a0ac 2024-11-10T03:19:15,451 DEBUG [StoreCloser-TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/c4970472e33446819b39dbf2662298a5 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/c4970472e33446819b39dbf2662298a5 2024-11-10T03:19:15,452 DEBUG [StoreCloser-TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/abc3e04fcd244982880cda8c79b4cb7d to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/abc3e04fcd244982880cda8c79b4cb7d 2024-11-10T03:19:15,452 DEBUG [StoreCloser-TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/c377e74ec4ce4f53b86de58c3056d232 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/c377e74ec4ce4f53b86de58c3056d232 2024-11-10T03:19:15,453 DEBUG [StoreCloser-TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/9bd9781a72784d23bb675e83a9b4f1ba to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/9bd9781a72784d23bb675e83a9b4f1ba 2024-11-10T03:19:15,453 DEBUG [StoreCloser-TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/ac10ffb12c06408288e9228e0fdbc7b6 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/ac10ffb12c06408288e9228e0fdbc7b6 2024-11-10T03:19:15,454 DEBUG [StoreCloser-TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/cd5f885188244e999b4e6c5a7ee5a1cb to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/cd5f885188244e999b4e6c5a7ee5a1cb 2024-11-10T03:19:15,455 DEBUG [StoreCloser-TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/ecf1d40a3d184a25b663072129b6f03c to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/ecf1d40a3d184a25b663072129b6f03c 2024-11-10T03:19:15,455 DEBUG [StoreCloser-TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/5e543f13bf214a2686f12ee81715b6f7 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/5e543f13bf214a2686f12ee81715b6f7 2024-11-10T03:19:15,456 DEBUG [StoreCloser-TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/9f8be0e486f5477fbbc72c5145e6a868 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/9f8be0e486f5477fbbc72c5145e6a868 2024-11-10T03:19:15,457 DEBUG [StoreCloser-TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/255a2df4142745d7bedb4a721a1e64a3 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/255a2df4142745d7bedb4a721a1e64a3 2024-11-10T03:19:15,457 DEBUG [StoreCloser-TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/5d8e8e9a550a434794c74a709812232e to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/5d8e8e9a550a434794c74a709812232e 2024-11-10T03:19:15,458 DEBUG [StoreCloser-TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/2f23dd9091c54eb79f3a67340b32f0dc to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/2f23dd9091c54eb79f3a67340b32f0dc 2024-11-10T03:19:15,459 DEBUG [StoreCloser-TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/883ceba6a39a4c32879c19f0efbfe667, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/c77c98d7cc2d498f9b2ffa442c14daed, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/4c4aa8792d714e2bb6f8dc31d4745074, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/5b7b606fd1444ddca74f08e77fe91496, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/6881928e13984d9bbf564f012d2f729c, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/cd75bf348ab94f5ab0b4bf8d15d4770c, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/d18082c3579b489ab025dc5446509982, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/923469d6fc70494b90c3fb4b9e791348, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/d4e9e04dafeb4e63ace5b0879c745b2c, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/914a8ec541274fcc9b8f212bc1c61d8c, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/5a2116a0c4364a12b011d795caf31789, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/8a344ce12dbd4c11a98afc3ce5351a8c, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/c380fc8373304f088f44503c03e34689, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/46e735407cfe40f1b3cf447f524dade9, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/6c9a604eebb24e97ab2df45b743e75fc, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/7aefeb063ed94d658f88bf62ddbcf0e9, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/9fb2fa6ce3554e2d8a1cab39114a760c, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/a1a5884f6c884c26bccc421cd31011fd, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/1e758f0108ae4175a0e6d4987291398b, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/86b8f45779164e73abcb3ef7c1b466bd, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/71623c6621d04f76ba2f74216420835a, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/eaa6fc324897441eb6a07300409f704a, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/f120f452952d4a829d7c5126db8bf8ac, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/187590c88f2247738acbfa0f375d3be5, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/93eacede815c49649faa2934879ce7d2] to archive 2024-11-10T03:19:15,459 DEBUG [StoreCloser-TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T03:19:15,460 DEBUG [StoreCloser-TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/883ceba6a39a4c32879c19f0efbfe667 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/883ceba6a39a4c32879c19f0efbfe667 2024-11-10T03:19:15,461 DEBUG [StoreCloser-TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/c77c98d7cc2d498f9b2ffa442c14daed to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/c77c98d7cc2d498f9b2ffa442c14daed 2024-11-10T03:19:15,461 DEBUG [StoreCloser-TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/4c4aa8792d714e2bb6f8dc31d4745074 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/4c4aa8792d714e2bb6f8dc31d4745074 2024-11-10T03:19:15,462 DEBUG [StoreCloser-TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/5b7b606fd1444ddca74f08e77fe91496 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/5b7b606fd1444ddca74f08e77fe91496 2024-11-10T03:19:15,462 DEBUG [StoreCloser-TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/6881928e13984d9bbf564f012d2f729c to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/6881928e13984d9bbf564f012d2f729c 2024-11-10T03:19:15,463 DEBUG [StoreCloser-TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/cd75bf348ab94f5ab0b4bf8d15d4770c to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/cd75bf348ab94f5ab0b4bf8d15d4770c 2024-11-10T03:19:15,464 DEBUG [StoreCloser-TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/d18082c3579b489ab025dc5446509982 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/d18082c3579b489ab025dc5446509982 2024-11-10T03:19:15,464 DEBUG [StoreCloser-TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/923469d6fc70494b90c3fb4b9e791348 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/923469d6fc70494b90c3fb4b9e791348 2024-11-10T03:19:15,465 DEBUG [StoreCloser-TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/d4e9e04dafeb4e63ace5b0879c745b2c to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/d4e9e04dafeb4e63ace5b0879c745b2c 2024-11-10T03:19:15,466 DEBUG [StoreCloser-TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/914a8ec541274fcc9b8f212bc1c61d8c to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/914a8ec541274fcc9b8f212bc1c61d8c 2024-11-10T03:19:15,466 DEBUG [StoreCloser-TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/5a2116a0c4364a12b011d795caf31789 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/5a2116a0c4364a12b011d795caf31789 2024-11-10T03:19:15,467 DEBUG [StoreCloser-TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/8a344ce12dbd4c11a98afc3ce5351a8c to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/8a344ce12dbd4c11a98afc3ce5351a8c 2024-11-10T03:19:15,468 DEBUG [StoreCloser-TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/c380fc8373304f088f44503c03e34689 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/c380fc8373304f088f44503c03e34689 2024-11-10T03:19:15,469 DEBUG [StoreCloser-TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/46e735407cfe40f1b3cf447f524dade9 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/46e735407cfe40f1b3cf447f524dade9 2024-11-10T03:19:15,469 DEBUG [StoreCloser-TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/6c9a604eebb24e97ab2df45b743e75fc to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/6c9a604eebb24e97ab2df45b743e75fc 2024-11-10T03:19:15,470 DEBUG [StoreCloser-TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/7aefeb063ed94d658f88bf62ddbcf0e9 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/7aefeb063ed94d658f88bf62ddbcf0e9 2024-11-10T03:19:15,471 DEBUG [StoreCloser-TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/9fb2fa6ce3554e2d8a1cab39114a760c to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/9fb2fa6ce3554e2d8a1cab39114a760c 2024-11-10T03:19:15,471 DEBUG [StoreCloser-TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/a1a5884f6c884c26bccc421cd31011fd to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/a1a5884f6c884c26bccc421cd31011fd 2024-11-10T03:19:15,472 DEBUG [StoreCloser-TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/1e758f0108ae4175a0e6d4987291398b to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/1e758f0108ae4175a0e6d4987291398b 2024-11-10T03:19:15,472 DEBUG [StoreCloser-TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/86b8f45779164e73abcb3ef7c1b466bd to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/86b8f45779164e73abcb3ef7c1b466bd 2024-11-10T03:19:15,473 DEBUG [StoreCloser-TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/71623c6621d04f76ba2f74216420835a to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/71623c6621d04f76ba2f74216420835a 2024-11-10T03:19:15,474 DEBUG [StoreCloser-TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/eaa6fc324897441eb6a07300409f704a to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/eaa6fc324897441eb6a07300409f704a 2024-11-10T03:19:15,474 DEBUG [StoreCloser-TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/f120f452952d4a829d7c5126db8bf8ac to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/f120f452952d4a829d7c5126db8bf8ac 2024-11-10T03:19:15,475 DEBUG [StoreCloser-TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/187590c88f2247738acbfa0f375d3be5 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/187590c88f2247738acbfa0f375d3be5 2024-11-10T03:19:15,476 DEBUG [StoreCloser-TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/93eacede815c49649faa2934879ce7d2 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/93eacede815c49649faa2934879ce7d2 2024-11-10T03:19:15,479 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/recovered.edits/363.seqid, newMaxSeqId=363, maxSeqId=4 2024-11-10T03:19:15,479 INFO [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. 2024-11-10T03:19:15,479 DEBUG [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1676): Region close journal for 07b306bf509fe2706c5b13e060d37e6b: Waiting for close lock at 1731208753776Running coprocessor pre-close hooks at 1731208753776Disabling compacts and flushes for region at 1731208753776Disabling writes for close at 1731208753777 (+1 ms)Obtaining lock to block concurrent updates at 1731208753777Preparing flush snapshotting stores in 07b306bf509fe2706c5b13e060d37e6b at 1731208753777Finished memstore snapshotting TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=41520, getOffHeapSize=0, getCellsCount=300 at 1731208753777Flushing stores of TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b. at 1731208753777Flushing 07b306bf509fe2706c5b13e060d37e6b/A: creating writer at 1731208753778 (+1 ms)Flushing mob file 07b306bf509fe2706c5b13e060d37e6b/A: appending metadata at 1731208753781 (+3 ms)Flushing mob file 07b306bf509fe2706c5b13e060d37e6b/A: closing flushed file at 1731208753781Flushing 07b306bf509fe2706c5b13e060d37e6b/A: appending metadata at 1731208754188 (+407 ms)Flushing 07b306bf509fe2706c5b13e060d37e6b/A: closing flushed file at 1731208754188Flushing 07b306bf509fe2706c5b13e060d37e6b/B: creating writer at 1731208754595 (+407 ms)Flushing 07b306bf509fe2706c5b13e060d37e6b/B: appending metadata at 1731208754597 (+2 ms)Flushing 07b306bf509fe2706c5b13e060d37e6b/B: closing flushed file at 1731208754597Flushing 07b306bf509fe2706c5b13e060d37e6b/C: creating writer at 1731208755003 (+406 ms)Flushing 07b306bf509fe2706c5b13e060d37e6b/C: appending metadata at 1731208755005 (+2 ms)Flushing 07b306bf509fe2706c5b13e060d37e6b/C: closing flushed file at 1731208755005Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@63a31bf4: reopening flushed file at 1731208755411 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5aca962f: reopening flushed file at 1731208755414 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1f94fbe5: reopening flushed file at 1731208755417 (+3 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 07b306bf509fe2706c5b13e060d37e6b in 1643ms, sequenceid=360, compaction requested=true at 1731208755421 (+4 ms)Writing region close event to WAL at 1731208755477 (+56 ms)Running coprocessor post-close hooks at 1731208755479 (+2 ms)Closed at 1731208755479 2024-11-10T03:19:15,481 INFO [RS_CLOSE_REGION-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] handler.UnassignRegionHandler(157): Closed 07b306bf509fe2706c5b13e060d37e6b 2024-11-10T03:19:15,481 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=174 updating hbase:meta row=07b306bf509fe2706c5b13e060d37e6b, regionState=CLOSED 2024-11-10T03:19:15,483 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=175, ppid=174, state=RUNNABLE, hasLock=false; CloseRegionProcedure 07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 because future has completed 2024-11-10T03:19:15,484 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=175, resume processing ppid=174 2024-11-10T03:19:15,484 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=175, ppid=174, state=SUCCESS, hasLock=false; CloseRegionProcedure 07b306bf509fe2706c5b13e060d37e6b, server=394d2222f724,38139,1731208572054 in 1.8590 sec 2024-11-10T03:19:15,485 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=174, resume processing ppid=173 2024-11-10T03:19:15,485 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=174, ppid=173, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=07b306bf509fe2706c5b13e060d37e6b, UNASSIGN in 1.8630 sec 2024-11-10T03:19:15,487 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=173, resume processing ppid=172 2024-11-10T03:19:15,487 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=173, ppid=172, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8660 sec 2024-11-10T03:19:15,488 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731208755488"}]},"ts":"1731208755488"} 2024-11-10T03:19:15,489 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-10T03:19:15,489 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-10T03:19:15,490 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=172, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8730 sec 2024-11-10T03:19:15,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-10T03:19:15,763 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-10T03:19:15,764 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-10T03:19:15,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] procedure2.ProcedureExecutor(1139): Stored pid=176, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T03:19:15,765 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=176, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T03:19:15,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-10T03:19:15,766 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=176, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T03:19:15,767 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b 2024-11-10T03:19:15,769 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A, FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B, FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C, FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/recovered.edits] 2024-11-10T03:19:15,772 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/3f20212038bd439c8798af36a0c9a1ba to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/3f20212038bd439c8798af36a0c9a1ba 2024-11-10T03:19:15,773 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/47e6bbda94064bc696411c4325871786 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/47e6bbda94064bc696411c4325871786 2024-11-10T03:19:15,774 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/9bbd33d092a444c7b5c63e26ea84a5d9 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/A/9bbd33d092a444c7b5c63e26ea84a5d9 2024-11-10T03:19:15,775 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/4cd9e2afaaa7455499b908fb0cd0bc32 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/4cd9e2afaaa7455499b908fb0cd0bc32 2024-11-10T03:19:15,776 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/755ecccb055a4c72af363ee6ebd33aff to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/755ecccb055a4c72af363ee6ebd33aff 2024-11-10T03:19:15,776 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/dbf735c0f71943dcb3c9c2c20eb62efb to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/B/dbf735c0f71943dcb3c9c2c20eb62efb 2024-11-10T03:19:15,778 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/61a4c4eea80c4632aeccf688e7610f95 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/61a4c4eea80c4632aeccf688e7610f95 2024-11-10T03:19:15,778 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/84cc219363104a399ee5a4fd8c28f516 to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/84cc219363104a399ee5a4fd8c28f516 2024-11-10T03:19:15,779 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/d172de31786241c5b3ada95ddc783c7a to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/C/d172de31786241c5b3ada95ddc783c7a 2024-11-10T03:19:15,780 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/recovered.edits/363.seqid to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b/recovered.edits/363.seqid 2024-11-10T03:19:15,781 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/default/TestAcidGuarantees/07b306bf509fe2706c5b13e060d37e6b 2024-11-10T03:19:15,781 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-10T03:19:15,781 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-10T03:19:15,782 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-10T03:19:15,783 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411100416aea5e8b94445987c1d13c1a4cf0b_07b306bf509fe2706c5b13e060d37e6b to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411100416aea5e8b94445987c1d13c1a4cf0b_07b306bf509fe2706c5b13e060d37e6b 2024-11-10T03:19:15,784 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110177c790c5d2d4600bfed7a71ce546835_07b306bf509fe2706c5b13e060d37e6b to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110177c790c5d2d4600bfed7a71ce546835_07b306bf509fe2706c5b13e060d37e6b 2024-11-10T03:19:15,785 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411102b1251c9a545457c9b2c4190cc478943_07b306bf509fe2706c5b13e060d37e6b to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411102b1251c9a545457c9b2c4190cc478943_07b306bf509fe2706c5b13e060d37e6b 2024-11-10T03:19:15,786 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411103116b00f72594f3aafcfa5fd4cf5617e_07b306bf509fe2706c5b13e060d37e6b to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411103116b00f72594f3aafcfa5fd4cf5617e_07b306bf509fe2706c5b13e060d37e6b 2024-11-10T03:19:15,787 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111033b52ebbc5da49518362d3eade86990a_07b306bf509fe2706c5b13e060d37e6b to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111033b52ebbc5da49518362d3eade86990a_07b306bf509fe2706c5b13e060d37e6b 2024-11-10T03:19:15,787 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411103964da02837a4bd793dae4b1534b88fc_07b306bf509fe2706c5b13e060d37e6b to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411103964da02837a4bd793dae4b1534b88fc_07b306bf509fe2706c5b13e060d37e6b 2024-11-10T03:19:15,788 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411103d1ea80acb2f478882e709e4f34524aa_07b306bf509fe2706c5b13e060d37e6b to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411103d1ea80acb2f478882e709e4f34524aa_07b306bf509fe2706c5b13e060d37e6b 2024-11-10T03:19:15,788 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111049c8d7284f7748d9b421309a1ebca726_07b306bf509fe2706c5b13e060d37e6b to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111049c8d7284f7748d9b421309a1ebca726_07b306bf509fe2706c5b13e060d37e6b 2024-11-10T03:19:15,789 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411105b2cd91846c04c348ffc2d0b9d4bee6c_07b306bf509fe2706c5b13e060d37e6b to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411105b2cd91846c04c348ffc2d0b9d4bee6c_07b306bf509fe2706c5b13e060d37e6b 2024-11-10T03:19:15,790 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111064f1c054c46c4af197975388095cb66e_07b306bf509fe2706c5b13e060d37e6b to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111064f1c054c46c4af197975388095cb66e_07b306bf509fe2706c5b13e060d37e6b 2024-11-10T03:19:15,790 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111070757c4084e744c68cea3b759f33d9c7_07b306bf509fe2706c5b13e060d37e6b to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111070757c4084e744c68cea3b759f33d9c7_07b306bf509fe2706c5b13e060d37e6b 2024-11-10T03:19:15,791 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111072d171f7377c4adf9803995cd510ada6_07b306bf509fe2706c5b13e060d37e6b to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111072d171f7377c4adf9803995cd510ada6_07b306bf509fe2706c5b13e060d37e6b 2024-11-10T03:19:15,791 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110756e0cb317254f63b2324e88e8cb2e6f_07b306bf509fe2706c5b13e060d37e6b to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110756e0cb317254f63b2324e88e8cb2e6f_07b306bf509fe2706c5b13e060d37e6b 2024-11-10T03:19:15,792 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111084849330c3a646f7bac56f00b4fa8f9c_07b306bf509fe2706c5b13e060d37e6b to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111084849330c3a646f7bac56f00b4fa8f9c_07b306bf509fe2706c5b13e060d37e6b 2024-11-10T03:19:15,793 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111084e7f10886084f61b577aa7dfdf902d2_07b306bf509fe2706c5b13e060d37e6b to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111084e7f10886084f61b577aa7dfdf902d2_07b306bf509fe2706c5b13e060d37e6b 2024-11-10T03:19:15,793 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411109e546f1fd0594cb29bb04e1ebb126ffc_07b306bf509fe2706c5b13e060d37e6b to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411109e546f1fd0594cb29bb04e1ebb126ffc_07b306bf509fe2706c5b13e060d37e6b 2024-11-10T03:19:15,794 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110c0a3337544a64bdf8d4f4522e483acf2_07b306bf509fe2706c5b13e060d37e6b to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110c0a3337544a64bdf8d4f4522e483acf2_07b306bf509fe2706c5b13e060d37e6b 2024-11-10T03:19:15,794 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110c37255c2b6eb4ae9bbd980f28c21def1_07b306bf509fe2706c5b13e060d37e6b to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110c37255c2b6eb4ae9bbd980f28c21def1_07b306bf509fe2706c5b13e060d37e6b 2024-11-10T03:19:15,795 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110cb67ebdc18de45af9e001d883d16fac2_07b306bf509fe2706c5b13e060d37e6b to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110cb67ebdc18de45af9e001d883d16fac2_07b306bf509fe2706c5b13e060d37e6b 2024-11-10T03:19:15,795 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110d8d7029b0d984bb790764720c6457e17_07b306bf509fe2706c5b13e060d37e6b to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110d8d7029b0d984bb790764720c6457e17_07b306bf509fe2706c5b13e060d37e6b 2024-11-10T03:19:15,796 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110ff3ffca112ef4679948dbd7876dddd0d_07b306bf509fe2706c5b13e060d37e6b to hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110ff3ffca112ef4679948dbd7876dddd0d_07b306bf509fe2706c5b13e060d37e6b 2024-11-10T03:19:15,796 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-10T03:19:15,798 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=176, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T03:19:15,799 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-10T03:19:15,801 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-10T03:19:15,801 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=176, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T03:19:15,802 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-10T03:19:15,802 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731208755802"}]},"ts":"9223372036854775807"} 2024-11-10T03:19:15,803 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-10T03:19:15,803 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 07b306bf509fe2706c5b13e060d37e6b, NAME => 'TestAcidGuarantees,,1731208725822.07b306bf509fe2706c5b13e060d37e6b.', STARTKEY => '', ENDKEY => ''}] 2024-11-10T03:19:15,803 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-10T03:19:15,803 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731208755803"}]},"ts":"9223372036854775807"} 2024-11-10T03:19:15,804 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-10T03:19:15,805 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=176, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T03:19:15,806 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=176, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 41 msec 2024-11-10T03:19:15,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37299 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-10T03:19:15,873 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-10T03:19:15,873 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-10T03:19:15,881 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=248 (was 246) - Thread LEAK? -, OpenFileDescriptor=447 (was 443) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=621 (was 683), ProcessCount=11 (was 11), AvailableMemoryMB=857 (was 1345) 2024-11-10T03:19:15,882 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-10T03:19:15,882 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T03:19:15,882 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T03:19:15,882 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:19:15,882 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:19:15,882 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T03:19:15,882 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-10T03:19:15,882 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=694036216, stopped=false 2024-11-10T03:19:15,883 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=394d2222f724,37299,1731208571229 2024-11-10T03:19:15,892 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38139-0x10122be61aa0001, quorum=127.0.0.1:58617, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-10T03:19:15,892 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37299-0x10122be61aa0000, quorum=127.0.0.1:58617, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-10T03:19:15,892 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37299-0x10122be61aa0000, quorum=127.0.0.1:58617, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T03:19:15,892 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38139-0x10122be61aa0001, quorum=127.0.0.1:58617, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T03:19:15,893 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-10T03:19:15,893 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T03:19:15,893 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T03:19:15,893 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:19:15,893 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:37299-0x10122be61aa0000, quorum=127.0.0.1:58617, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T03:19:15,893 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38139-0x10122be61aa0001, quorum=127.0.0.1:58617, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T03:19:15,894 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '394d2222f724,38139,1731208572054' ***** 2024-11-10T03:19:15,894 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-10T03:19:15,894 INFO [RS:0;394d2222f724:38139 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-10T03:19:15,894 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-10T03:19:15,894 INFO [RS:0;394d2222f724:38139 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-10T03:19:15,894 INFO [RS:0;394d2222f724:38139 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-10T03:19:15,895 INFO [RS:0;394d2222f724:38139 {}] regionserver.HRegionServer(959): stopping server 394d2222f724,38139,1731208572054 2024-11-10T03:19:15,895 INFO [RS:0;394d2222f724:38139 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-10T03:19:15,895 INFO [RS:0;394d2222f724:38139 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;394d2222f724:38139. 2024-11-10T03:19:15,895 DEBUG [RS:0;394d2222f724:38139 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T03:19:15,895 DEBUG [RS:0;394d2222f724:38139 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:19:15,895 INFO [RS:0;394d2222f724:38139 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-10T03:19:15,895 INFO [RS:0;394d2222f724:38139 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-10T03:19:15,895 INFO [RS:0;394d2222f724:38139 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-10T03:19:15,895 INFO [RS:0;394d2222f724:38139 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-10T03:19:15,896 INFO [RS:0;394d2222f724:38139 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-10T03:19:15,896 DEBUG [RS:0;394d2222f724:38139 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-10T03:19:15,896 DEBUG [RS_CLOSE_META-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-10T03:19:15,896 DEBUG [RS:0;394d2222f724:38139 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-10T03:19:15,896 INFO [RS_CLOSE_META-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-10T03:19:15,896 DEBUG [RS_CLOSE_META-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-10T03:19:15,896 DEBUG [RS_CLOSE_META-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-10T03:19:15,896 DEBUG [RS_CLOSE_META-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-10T03:19:15,896 INFO [RS_CLOSE_META-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-10T03:19:15,918 DEBUG [RS_CLOSE_META-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/hbase/meta/1588230740/.tmp/info/76f76c0c82c74b7786d99f9f1d090b0f is 95, key is TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722./info:/1731208601183/DeleteFamily/seqid=0 2024-11-10T03:19:15,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742492_1668 (size=6514) 2024-11-10T03:19:15,965 INFO [regionserver/394d2222f724:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-10T03:19:16,096 DEBUG [RS:0;394d2222f724:38139 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-10T03:19:16,269 INFO [regionserver/394d2222f724:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-10T03:19:16,269 INFO [regionserver/394d2222f724:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-10T03:19:16,296 DEBUG [RS:0;394d2222f724:38139 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-10T03:19:16,323 INFO [RS_CLOSE_META-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/hbase/meta/1588230740/.tmp/info/76f76c0c82c74b7786d99f9f1d090b0f 2024-11-10T03:19:16,338 DEBUG [RS_CLOSE_META-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/hbase/meta/1588230740/.tmp/ns/1e15b36dc4174f7d8adfa7841f0f4da0 is 93, key is TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722./ns:/1731208601141/DeleteFamily/seqid=0 2024-11-10T03:19:16,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742493_1669 (size=5906) 2024-11-10T03:19:16,497 DEBUG [RS:0;394d2222f724:38139 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-10T03:19:16,697 DEBUG [RS:0;394d2222f724:38139 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-10T03:19:16,742 INFO [RS_CLOSE_META-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/hbase/meta/1588230740/.tmp/ns/1e15b36dc4174f7d8adfa7841f0f4da0 2024-11-10T03:19:16,758 DEBUG [RS_CLOSE_META-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/hbase/meta/1588230740/.tmp/rep_barrier/f381b498429148aa852f16902f4f7bb2 is 102, key is TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722./rep_barrier:/1731208601141/DeleteFamily/seqid=0 2024-11-10T03:19:16,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742494_1670 (size=6025) 2024-11-10T03:19:16,897 DEBUG [RS:0;394d2222f724:38139 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-10T03:19:17,097 DEBUG [RS:0;394d2222f724:38139 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-10T03:19:17,161 INFO [RS_CLOSE_META-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/hbase/meta/1588230740/.tmp/rep_barrier/f381b498429148aa852f16902f4f7bb2 2024-11-10T03:19:17,178 DEBUG [RS_CLOSE_META-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/hbase/meta/1588230740/.tmp/table/f3ee1dc7d19246d2bc4aaa112085ac97 is 96, key is TestAcidGuarantees,,1731208575848.77ed36df592326523261438e65e5b722./table:/1731208601141/DeleteFamily/seqid=0 2024-11-10T03:19:17,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742495_1671 (size=5936) 2024-11-10T03:19:17,297 DEBUG [RS:0;394d2222f724:38139 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-10T03:19:17,498 DEBUG [RS:0;394d2222f724:38139 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-10T03:19:17,581 INFO [RS_CLOSE_META-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/hbase/meta/1588230740/.tmp/table/f3ee1dc7d19246d2bc4aaa112085ac97 2024-11-10T03:19:17,584 DEBUG [RS_CLOSE_META-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/hbase/meta/1588230740/.tmp/info/76f76c0c82c74b7786d99f9f1d090b0f as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/hbase/meta/1588230740/info/76f76c0c82c74b7786d99f9f1d090b0f 2024-11-10T03:19:17,587 INFO [RS_CLOSE_META-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/hbase/meta/1588230740/info/76f76c0c82c74b7786d99f9f1d090b0f, entries=12, sequenceid=90, filesize=6.4 K 2024-11-10T03:19:17,588 DEBUG [RS_CLOSE_META-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/hbase/meta/1588230740/.tmp/ns/1e15b36dc4174f7d8adfa7841f0f4da0 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/hbase/meta/1588230740/ns/1e15b36dc4174f7d8adfa7841f0f4da0 2024-11-10T03:19:17,590 INFO [RS_CLOSE_META-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/hbase/meta/1588230740/ns/1e15b36dc4174f7d8adfa7841f0f4da0, entries=8, sequenceid=90, filesize=5.8 K 2024-11-10T03:19:17,590 DEBUG [RS_CLOSE_META-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/hbase/meta/1588230740/.tmp/rep_barrier/f381b498429148aa852f16902f4f7bb2 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/hbase/meta/1588230740/rep_barrier/f381b498429148aa852f16902f4f7bb2 2024-11-10T03:19:17,593 INFO [RS_CLOSE_META-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/hbase/meta/1588230740/rep_barrier/f381b498429148aa852f16902f4f7bb2, entries=6, sequenceid=90, filesize=5.9 K 2024-11-10T03:19:17,594 DEBUG [RS_CLOSE_META-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/hbase/meta/1588230740/.tmp/table/f3ee1dc7d19246d2bc4aaa112085ac97 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/hbase/meta/1588230740/table/f3ee1dc7d19246d2bc4aaa112085ac97 2024-11-10T03:19:17,597 INFO [RS_CLOSE_META-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/hbase/meta/1588230740/table/f3ee1dc7d19246d2bc4aaa112085ac97, entries=7, sequenceid=90, filesize=5.8 K 2024-11-10T03:19:17,598 INFO [RS_CLOSE_META-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1702ms, sequenceid=90, compaction requested=false 2024-11-10T03:19:17,601 DEBUG [RS_CLOSE_META-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-10T03:19:17,601 DEBUG [RS_CLOSE_META-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-10T03:19:17,601 INFO [RS_CLOSE_META-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-10T03:19:17,602 DEBUG [RS_CLOSE_META-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731208755896Running coprocessor pre-close hooks at 1731208755896Disabling compacts and flushes for region at 1731208755896Disabling writes for close at 1731208755896Obtaining lock to block concurrent updates at 1731208755896Preparing flush snapshotting stores in 1588230740 at 1731208755896Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1731208755897 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731208755898 (+1 ms)Flushing 1588230740/info: creating writer at 1731208755898Flushing 1588230740/info: appending metadata at 1731208755917 (+19 ms)Flushing 1588230740/info: closing flushed file at 1731208755918 (+1 ms)Flushing 1588230740/ns: creating writer at 1731208756326 (+408 ms)Flushing 1588230740/ns: appending metadata at 1731208756338 (+12 ms)Flushing 1588230740/ns: closing flushed file at 1731208756338Flushing 1588230740/rep_barrier: creating writer at 1731208756745 (+407 ms)Flushing 1588230740/rep_barrier: appending metadata at 1731208756758 (+13 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1731208756758Flushing 1588230740/table: creating writer at 1731208757164 (+406 ms)Flushing 1588230740/table: appending metadata at 1731208757177 (+13 ms)Flushing 1588230740/table: closing flushed file at 1731208757178 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2faae1e3: reopening flushed file at 1731208757584 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2e0fe071: reopening flushed file at 1731208757587 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@22ae09a5: reopening flushed file at 1731208757590 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@569f762a: reopening flushed file at 1731208757594 (+4 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1702ms, sequenceid=90, compaction requested=false at 1731208757598 (+4 ms)Writing region close event to WAL at 1731208757599 (+1 ms)Running coprocessor post-close hooks at 1731208757601 (+2 ms)Closed at 1731208757601 2024-11-10T03:19:17,602 DEBUG [RS_CLOSE_META-regionserver/394d2222f724:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-10T03:19:17,698 INFO [RS:0;394d2222f724:38139 {}] regionserver.HRegionServer(976): stopping server 394d2222f724,38139,1731208572054; all regions closed. 2024-11-10T03:19:17,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741834_1010 (size=25344) 2024-11-10T03:19:17,704 DEBUG [RS:0;394d2222f724:38139 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/oldWALs 2024-11-10T03:19:17,704 INFO [RS:0;394d2222f724:38139 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 394d2222f724%2C38139%2C1731208572054.meta:.meta(num 1731208575391) 2024-11-10T03:19:17,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741832_1008 (size=17916197) 2024-11-10T03:19:17,708 DEBUG [RS:0;394d2222f724:38139 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/oldWALs 2024-11-10T03:19:17,708 INFO [RS:0;394d2222f724:38139 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 394d2222f724%2C38139%2C1731208572054:(num 1731208574433) 2024-11-10T03:19:17,708 DEBUG [RS:0;394d2222f724:38139 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T03:19:17,708 INFO [RS:0;394d2222f724:38139 {}] regionserver.LeaseManager(133): Closed leases 2024-11-10T03:19:17,708 INFO [RS:0;394d2222f724:38139 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-10T03:19:17,708 INFO [RS:0;394d2222f724:38139 {}] hbase.ChoreService(370): Chore service for: regionserver/394d2222f724:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-10T03:19:17,708 INFO [RS:0;394d2222f724:38139 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-10T03:19:17,708 INFO [regionserver/394d2222f724:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-10T03:19:17,709 INFO [RS:0;394d2222f724:38139 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:38139 2024-11-10T03:19:17,724 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37299-0x10122be61aa0000, quorum=127.0.0.1:58617, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-10T03:19:17,724 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38139-0x10122be61aa0001, quorum=127.0.0.1:58617, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/394d2222f724,38139,1731208572054 2024-11-10T03:19:17,724 INFO [RS:0;394d2222f724:38139 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-10T03:19:17,725 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [394d2222f724,38139,1731208572054] 2024-11-10T03:19:17,734 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/394d2222f724,38139,1731208572054 already deleted, retry=false 2024-11-10T03:19:17,735 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 394d2222f724,38139,1731208572054 expired; onlineServers=0 2024-11-10T03:19:17,735 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '394d2222f724,37299,1731208571229' ***** 2024-11-10T03:19:17,735 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-10T03:19:17,735 INFO [M:0;394d2222f724:37299 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-10T03:19:17,735 INFO [M:0;394d2222f724:37299 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-10T03:19:17,735 DEBUG [M:0;394d2222f724:37299 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-10T03:19:17,735 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-10T03:19:17,735 DEBUG [M:0;394d2222f724:37299 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-10T03:19:17,735 DEBUG [master/394d2222f724:0:becomeActiveMaster-HFileCleaner.small.0-1731208574066 {}] cleaner.HFileCleaner(306): Exit Thread[master/394d2222f724:0:becomeActiveMaster-HFileCleaner.small.0-1731208574066,5,FailOnTimeoutGroup] 2024-11-10T03:19:17,735 DEBUG [master/394d2222f724:0:becomeActiveMaster-HFileCleaner.large.0-1731208574065 {}] cleaner.HFileCleaner(306): Exit Thread[master/394d2222f724:0:becomeActiveMaster-HFileCleaner.large.0-1731208574065,5,FailOnTimeoutGroup] 2024-11-10T03:19:17,735 INFO [M:0;394d2222f724:37299 {}] hbase.ChoreService(370): Chore service for: master/394d2222f724:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-10T03:19:17,736 INFO [M:0;394d2222f724:37299 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-10T03:19:17,736 DEBUG [M:0;394d2222f724:37299 {}] master.HMaster(1795): Stopping service threads 2024-11-10T03:19:17,736 INFO [M:0;394d2222f724:37299 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-10T03:19:17,736 INFO [M:0;394d2222f724:37299 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-10T03:19:17,736 ERROR [M:0;394d2222f724:37299 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-10T03:19:17,736 INFO [M:0;394d2222f724:37299 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-10T03:19:17,736 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-10T03:19:17,745 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37299-0x10122be61aa0000, quorum=127.0.0.1:58617, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-10T03:19:17,745 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37299-0x10122be61aa0000, quorum=127.0.0.1:58617, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T03:19:17,745 DEBUG [M:0;394d2222f724:37299 {}] zookeeper.ZKUtil(347): master:37299-0x10122be61aa0000, quorum=127.0.0.1:58617, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-10T03:19:17,745 WARN [M:0;394d2222f724:37299 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-10T03:19:17,746 INFO [M:0;394d2222f724:37299 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/.lastflushedseqids 2024-11-10T03:19:17,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742496_1672 (size=72) 2024-11-10T03:19:17,835 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38139-0x10122be61aa0001, quorum=127.0.0.1:58617, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T03:19:17,835 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38139-0x10122be61aa0001, quorum=127.0.0.1:58617, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T03:19:17,835 INFO [RS:0;394d2222f724:38139 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-10T03:19:17,836 INFO [RS:0;394d2222f724:38139 {}] regionserver.HRegionServer(1031): Exiting; stopping=394d2222f724,38139,1731208572054; zookeeper connection closed. 2024-11-10T03:19:17,836 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@19de7127 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@19de7127 2024-11-10T03:19:17,836 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-10T03:19:18,154 INFO [M:0;394d2222f724:37299 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-10T03:19:18,154 INFO [M:0;394d2222f724:37299 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-10T03:19:18,154 DEBUG [M:0;394d2222f724:37299 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-10T03:19:18,154 INFO [M:0;394d2222f724:37299 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T03:19:18,154 DEBUG [M:0;394d2222f724:37299 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T03:19:18,154 DEBUG [M:0;394d2222f724:37299 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-10T03:19:18,154 DEBUG [M:0;394d2222f724:37299 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T03:19:18,154 INFO [M:0;394d2222f724:37299 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=797.83 KB heapSize=979.58 KB 2024-11-10T03:19:18,168 DEBUG [M:0;394d2222f724:37299 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2342b12a6ab84bf8addc7a2fdcfe55a8 is 82, key is hbase:meta,,1/info:regioninfo/1731208575493/Put/seqid=0 2024-11-10T03:19:18,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742497_1673 (size=5672) 2024-11-10T03:19:18,571 INFO [M:0;394d2222f724:37299 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2245 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2342b12a6ab84bf8addc7a2fdcfe55a8 2024-11-10T03:19:18,590 DEBUG [M:0;394d2222f724:37299 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/4aa963636f7740489850bc165f005176 is 2479, key is \x00\x00\x00\x00\x00\x00\x00\x95/proc:d/1731208728886/Put/seqid=0 2024-11-10T03:19:18,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742498_1674 (size=46396) 2024-11-10T03:19:18,994 INFO [M:0;394d2222f724:37299 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=797.28 KB at sequenceid=2245 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/4aa963636f7740489850bc165f005176 2024-11-10T03:19:18,997 INFO [M:0;394d2222f724:37299 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 4aa963636f7740489850bc165f005176 2024-11-10T03:19:19,011 DEBUG [M:0;394d2222f724:37299 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/0d4e037b720d47cdb82576baf76ef031 is 69, key is 394d2222f724,38139,1731208572054/rs:state/1731208574186/Put/seqid=0 2024-11-10T03:19:19,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073742499_1675 (size=5156) 2024-11-10T03:19:19,414 INFO [M:0;394d2222f724:37299 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2245 (bloomFilter=true), to=hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/0d4e037b720d47cdb82576baf76ef031 2024-11-10T03:19:19,418 DEBUG [M:0;394d2222f724:37299 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2342b12a6ab84bf8addc7a2fdcfe55a8 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/2342b12a6ab84bf8addc7a2fdcfe55a8 2024-11-10T03:19:19,421 INFO [M:0;394d2222f724:37299 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/2342b12a6ab84bf8addc7a2fdcfe55a8, entries=8, sequenceid=2245, filesize=5.5 K 2024-11-10T03:19:19,422 DEBUG [M:0;394d2222f724:37299 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/4aa963636f7740489850bc165f005176 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/4aa963636f7740489850bc165f005176 2024-11-10T03:19:19,425 INFO [M:0;394d2222f724:37299 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 4aa963636f7740489850bc165f005176 2024-11-10T03:19:19,425 INFO [M:0;394d2222f724:37299 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/4aa963636f7740489850bc165f005176, entries=176, sequenceid=2245, filesize=45.3 K 2024-11-10T03:19:19,425 DEBUG [M:0;394d2222f724:37299 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/0d4e037b720d47cdb82576baf76ef031 as hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/0d4e037b720d47cdb82576baf76ef031 2024-11-10T03:19:19,428 INFO [M:0;394d2222f724:37299 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46291/user/jenkins/test-data/f55b4352-f664-5bb9-03b6-446a6e2a1ff8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/0d4e037b720d47cdb82576baf76ef031, entries=1, sequenceid=2245, filesize=5.0 K 2024-11-10T03:19:19,429 INFO [M:0;394d2222f724:37299 {}] regionserver.HRegion(3140): Finished flush of dataSize ~797.83 KB/816979, heapSize ~979.28 KB/1002784, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1274ms, sequenceid=2245, compaction requested=false 2024-11-10T03:19:19,430 INFO [M:0;394d2222f724:37299 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T03:19:19,430 DEBUG [M:0;394d2222f724:37299 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731208758154Disabling compacts and flushes for region at 1731208758154Disabling writes for close at 1731208758154Obtaining lock to block concurrent updates at 1731208758154Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731208758154Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=816979, getHeapSize=1003024, getOffHeapSize=0, getCellsCount=2446 at 1731208758154Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731208758155 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731208758155Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731208758168 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731208758168Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731208758575 (+407 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731208758590 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731208758590Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731208758997 (+407 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731208759010 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731208759010Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7f2c546: reopening flushed file at 1731208759417 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@48582dca: reopening flushed file at 1731208759421 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@57f6301f: reopening flushed file at 1731208759425 (+4 ms)Finished flush of dataSize ~797.83 KB/816979, heapSize ~979.28 KB/1002784, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1274ms, sequenceid=2245, compaction requested=false at 1731208759429 (+4 ms)Writing region close event to WAL at 1731208759430 (+1 ms)Closed at 1731208759430 2024-11-10T03:19:19,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40501 is added to blk_1073741830_1006 (size=964127) 2024-11-10T03:19:19,433 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-10T03:19:19,433 INFO [M:0;394d2222f724:37299 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-10T03:19:19,433 INFO [M:0;394d2222f724:37299 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:37299 2024-11-10T03:19:19,433 INFO [M:0;394d2222f724:37299 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-10T03:19:19,540 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37299-0x10122be61aa0000, quorum=127.0.0.1:58617, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T03:19:19,540 INFO [M:0;394d2222f724:37299 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-10T03:19:19,540 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37299-0x10122be61aa0000, quorum=127.0.0.1:58617, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T03:19:19,549 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@46a94ab{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T03:19:19,551 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@753587b3{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-10T03:19:19,551 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-10T03:19:19,552 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4b5418b0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-10T03:19:19,552 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@24457f8e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0f0991c3-80e8-30ad-d408-9222f0ade083/hadoop.log.dir/,STOPPED} 2024-11-10T03:19:19,555 WARN [BP-1178049631-172.17.0.3-1731208565928 heartbeating to localhost/127.0.0.1:46291 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-10T03:19:19,555 WARN [BP-1178049631-172.17.0.3-1731208565928 heartbeating to localhost/127.0.0.1:46291 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1178049631-172.17.0.3-1731208565928 (Datanode Uuid 4301a539-d672-4639-8044-4a5372b056ca) service to localhost/127.0.0.1:46291 2024-11-10T03:19:19,556 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-10T03:19:19,556 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-10T03:19:19,557 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0f0991c3-80e8-30ad-d408-9222f0ade083/cluster_eaab6309-33cf-8146-ba76-8df954bdc4c1/data/data1/current/BP-1178049631-172.17.0.3-1731208565928 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T03:19:19,558 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0f0991c3-80e8-30ad-d408-9222f0ade083/cluster_eaab6309-33cf-8146-ba76-8df954bdc4c1/data/data2/current/BP-1178049631-172.17.0.3-1731208565928 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T03:19:19,558 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-10T03:19:19,569 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7f405bdd{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-10T03:19:19,569 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5b829fcb{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-10T03:19:19,569 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-10T03:19:19,570 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@31a6a643{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-10T03:19:19,570 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@610d6b27{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0f0991c3-80e8-30ad-d408-9222f0ade083/hadoop.log.dir/,STOPPED} 2024-11-10T03:19:19,589 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-10T03:19:19,718 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down